From 1f5436fe91fe77ef9306b1508d6a205c42d7a95e Mon Sep 17 00:00:00 2001 From: Lyndon-Li Date: Fri, 6 Jun 2025 17:22:08 +0800 Subject: [PATCH] vgdp ms pvr controller Signed-off-by: Lyndon-Li --- .../v1/bases/velero.io_podvolumerestores.yaml | 75 +- config/crd/v1/crds/crds.go | 2 +- pkg/apis/velero/v1/labels_annotations.go | 3 + pkg/apis/velero/v1/pod_volume_restore_type.go | 34 +- pkg/apis/velero/v1/zz_generated.deepcopy.go | 4 + pkg/builder/pod_builder.go | 5 + pkg/builder/pod_volume_restore_builder.go | 30 + pkg/cmd/cli/nodeagent/server.go | 10 +- pkg/controller/data_download_controller.go | 5 +- pkg/controller/data_upload_controller.go | 5 +- .../pod_volume_backup_controller.go | 5 +- .../pod_volume_restore_controller.go | 895 +++++++++++++---- .../pod_volume_restore_controller_legacy.go | 10 + .../pod_volume_restore_controller_test.go | 932 +++++++++++++++++- pkg/exposer/mocks/PodVolumeExposer.go | 125 +++ pkg/podvolume/restorer.go | 4 +- 16 files changed, 1901 insertions(+), 243 deletions(-) create mode 100644 pkg/controller/pod_volume_restore_controller_legacy.go create mode 100644 pkg/exposer/mocks/PodVolumeExposer.go diff --git a/config/crd/v1/bases/velero.io_podvolumerestores.yaml b/config/crd/v1/bases/velero.io_podvolumerestores.yaml index 888ac1669..09eda5b28 100644 --- a/config/crd/v1/bases/velero.io_podvolumerestores.yaml +++ b/config/crd/v1/bases/velero.io_podvolumerestores.yaml @@ -15,39 +15,40 @@ spec: scope: Namespaced versions: - additionalPrinterColumns: - - description: Namespace of the pod containing the volume to be restored - jsonPath: .spec.pod.namespace - name: Namespace + - description: PodVolumeRestore status such as New/InProgress + jsonPath: .status.phase + name: Status type: string - - description: Name of the pod containing the volume to be restored - jsonPath: .spec.pod.name - name: Pod + - description: Time duration since this PodVolumeRestore was started + jsonPath: .status.startTimestamp + name: Started + type: date + - description: Completed bytes + format: int64 + jsonPath: .status.progress.bytesDone + name: Bytes Done + type: integer + - description: Total bytes + format: int64 + jsonPath: .status.progress.totalBytes + name: Total Bytes + type: integer + - description: Name of the Backup Storage Location where the backup data is stored + jsonPath: .spec.backupStorageLocation + name: Storage Location + type: string + - description: Time duration since this PodVolumeRestore was created + jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Name of the node where the PodVolumeRestore is processed + jsonPath: .status.node + name: Node type: string - description: The type of the uploader to handle data transfer jsonPath: .spec.uploaderType name: Uploader Type type: string - - description: Name of the volume to be restored - jsonPath: .spec.volume - name: Volume - type: string - - description: Pod Volume Restore status such as New/InProgress - jsonPath: .status.phase - name: Status - type: string - - description: Pod Volume Restore status such as New/InProgress - format: int64 - jsonPath: .status.progress.totalBytes - name: TotalBytes - type: integer - - description: Pod Volume Restore status such as New/InProgress - format: int64 - jsonPath: .status.progress.bytesDone - name: BytesDone - type: integer - - jsonPath: .metadata.creationTimestamp - name: Age - type: date name: v1 schema: openAPIV3Schema: @@ -77,6 +78,11 @@ spec: BackupStorageLocation is the name of the backup storage location where the backup repository is stored. type: string + cancel: + description: |- + Cancel indicates request to cancel the ongoing PodVolumeRestore. It can be set + when the PodVolumeRestore is in InProgress phase + type: boolean pod: description: Pod is a reference to the pod containing the volume to be restored. @@ -162,6 +168,13 @@ spec: status: description: PodVolumeRestoreStatus is the current status of a PodVolumeRestore. properties: + acceptedTimestamp: + description: |- + AcceptedTimestamp records the time the pod volume restore is to be prepared. + The server's time is used for AcceptedTimestamp + format: date-time + nullable: true + type: string completionTimestamp: description: |- CompletionTimestamp records the time a restore was completed. @@ -173,11 +186,19 @@ spec: message: description: Message is a message about the pod volume restore's status. type: string + node: + description: Node is name of the node where the pod volume restore + is processed. + type: string phase: description: Phase is the current state of the PodVolumeRestore. enum: - New + - Accepted + - Prepared - InProgress + - Canceling + - Canceled - Completed - Failed type: string diff --git a/config/crd/v1/crds/crds.go b/config/crd/v1/crds/crds.go index a412073f6..0288fbcce 100644 --- a/config/crd/v1/crds/crds.go +++ b/config/crd/v1/crds/crds.go @@ -35,7 +35,7 @@ var rawCRDs = [][]byte{ []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcVMo\x1b7\x10\xbd\xebW\f\xd0kwU\xa3hQ\xec\xadqr0\xda\x06\x82\x1d\xe4N\x91#-c.\xc9\xce\f\xe5\xba\x1f\xff\xbd \xb9+K\xab\x95\x93\\\xb27\x91Ù\xc7\xf7f\x1e\xd54\xcdJE\xfb\x11\x89m\xf0\x1d\xa8h\xf1/A\x9f\x7fq\xfb\xf8\v\xb76\xac\x0f7\xabG\xebM\a\xb7\x89%\f\xf7\xc8!\x91Ʒ\xb8\xb3ފ\r~5\xa0(\xa3Du+\x00\xe5}\x10\x95\x979\xff\x04\xd0\xc1\v\x05琚=\xfa\xf61mq\x9b\xac3H%\xf9T\xfa\xf0C{\xf3s\xfb\xd3\n\xc0\xab\x01;0\xe8Pp\xab\xf4c\x8a\x84\x7f&d\xe1\xf6\x80\x0e)\xb46\xac8\xa2\xce\xf9\xf7\x14R\xec\xe0e\xa3\x9e\x1fkW\xdcoK\xaa7%\xd5}MUv\x9de\xf9\xedZ\xc4\xefv\x8c\x8a.\x91rˀJ\x00[\xbfON\xd1b\xc8\n\x80u\x88\xd8\xc1\xfb\f+*\x8df\x050^\xbb\xc0l@\x19S\x88TnC\xd6\v\xd2mpi\x98\bl\xc0 k\xb2Q\nQ\x1fz,W\x84\xb0\x03\xe9\x11j9\x90\x00[\x1c\x11\x98r\x0e\xe0\x13\a\xbfQ\xd2w\xd0f\xbe\xda\x1a\x9a\x81\x8c\x01\x95\xea7\xf3ey\u0380Y\xc8\xfa\xfd5\b,J\x12O J]\x1b<\xd0\t\xbf\xe7\x00J|\x1b{\xc5\xe7\xd5\x1f\xcaƵ\xca5\xe6pS\x99\xd6=\x0e\xaa\x1bcCD\xff\xeb\xe6\xee\xe3\x8f\x0fg\xcbp\x8euAZ\xb0\fjB\x9a\x89\xab\xacA\xf0\b\x81`\b4\xb1\xca\xed1i\xa4\x10\x91\xc4N\xadU\xbf\x93\xe19Y\x9dA\xf8\xb79\xdb\x03Ȩ\xeb)0y\x8a\x90\v\x89cS\xa0\x19/Zɵ\f\x84\x91\x90\xd1\u05f9\xca\xcb\xcaC\xd8~B-\xed,\xf5\x03RN\x03܇\xe4L\x1e\xbe\x03\x92\x00\xa1\x0e{o\xff>\xe6\xe6|\xef\\\xd4))\x94\xe4\xb6\xf3\xca\xc1A\xb9\x84߃\xf2f\x96yP\xcf@\x98kB\xf2'\xf9\xca\x01\x9e\xe3\xf8#\x93h\xfd.tЋD\xee\xd6뽕\xc9Rt\x18\x86\xe4\xad<\xaf\x8b;\xd8m\x92@\xbc6x@\xb7f\xbbo\x14\xe9\xde\njI\x84k\x15mS.⋭\xb4\x83\xf9\x8eF\x13Ⳳ\x17\xddS\xbf\xe2\x02_!O\xf6\x84\xda#5U\xbd\xe2\x8b\ny)Sw\xff\xee\xe1\x03LH\xaaRU\x94\x97\xd0\v^&}2\x9b\xd6\xef\x90\xea\xb9\x1d\x85\xa1\xe4Dob\xb0^\xca\x0f\xed,z\x01N\xdb\xc1\nO\x1d\x9b\xa5\x9b\xa7\xbd-\xb6\x9b\x1d E\xa3\x04\xcd<\xe0\xceí\x1a\xd0\xdd*\xc6o\xacUV\x85\x9b,\xc2\x17\xa9u\xfa\x98̃+\xbd'\x1b\xd33pEڅ\xe1\x7f\x88\xa8\xb3\xb8\x99\xdf|\xda\ueb2ec\xb5\v\x04O\xbd\xd5\xfd4\xfc3\x9a\x8eFq\xce߲1\xe4\xef\xc5n\xe7;W/\x0fEdK8k\xd8\x06.\xbc\xfbu^\x8a\xa9~%3\xd5\xd1Gnt\"*\xcdw\xf4y\xb5t\xe8K\xb9@\xa2@\x17\xab3P\xefJP\xf9Ǡ\xacgP\xfey<\b\xd2+\x81'\xa4\r\x97\x95\x1ax\x8fO\v\xabw~CaO\xc8\xf3\x96ϛ\x9b\xca\x1e\xce߃WXZlʋE\xceVhNXd\t\xa4\xf6\xa7\xbcr\xda\x1e\x9d\xbe\x83\x7f\xfe[\xfd\x1f\x00\x00\xff\xff\xbeM\x1a\xea\xb1\n\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcWMo\xe36\x10\xbd\xfbW\f\xd0K\v\xac\xe4\x06E\x8b·\xd6\xd9C\xb0\xe96\x88\xb7\xb9S\xd4HbC\x91,9t6E\x7f|1\xa4\xe4\x0fYv\x9c\xcb\xea\xe6\xe1p\xf8\xe6\xcd\xcc#]\x14\xc5B8\xf5\x84>(kV \x9c¯\x84\x86\x7f\x85\xf2\xf9\xd7P*\xbb\xdc\xde,\x9e\x95\xa9W\xb0\x8e\x81l\xff\x88\xc1F/\xf1\x16\x1be\x14)k\x16=\x92\xa8\x05\x89\xd5\x02@\x18cI\xb09\xf0O\x00i\ry\xab5\xfa\xa2ES>\xc7\n\xab\xa8t\x8d>\x05\x1f\x8f\xde\xfeX\xde\xfcR\xfe\xbc\x000\xa2\xc7\x15\xd4\xf6\xc5h+j\x8f\xffD\f\x14\xca-j\xf4\xb6Tv\x11\x1cJ\x8e\xddz\x1b\xdd\n\xf6\vy\xefpn\xc6|;\x84y\xccaҊV\x81>ͭޫ\xc1\xc3\xe9\xe8\x85>\x05\x91\x16\x832m\xd4\u009f,/\x00\x82\xb4\x0eW\xf0\x99a8!\xb1^\x00\f)&XŐ\xdd\xf6&\x87\x92\x1d\xf6\"\xe3\x05\xb0\x0e\xcdo\x0fwO?m\x8e\xcc\x005\x06镣D\xd4\x7f\xc5\xce\x0e\xd3\x04@\x05\x100\xc0\x01\xb2;\x84 \f\bO\xaa\x11\x92\xa0\xf1\xb6\x87J\xc8\xe7\xe8\xc0V\x7f\xa3$\bd\xbdh\xf1\x03\x84(;\x10\x1c%;\x1c\x9c\xa5m\v\x8d\xd2X\xeel\xce[\x87\x9e\xd4Hy\xfe\x0e\x1a\xea\xc0z)\v\xfe8\xf1\xbc\vj\xee,\f@\x1d\x8e\xe4a=p\x05\xb6\x01\xeaT\x00\x8f\xcec@\x93{\x8d\xcd\xc2\fٔ\x93\xd0\x1b\xf4\x1c\x06Bg\xa3\xae\xb9!\xb7\xe8\t\x1aE\xaf\xcb41\xaa\x8ad}XָE\xbd\f\xaa-\x84\x97\x9d\"\x94\x14=.\x85SEJĤQ+\xfb\xfa;?\ff8:\x96^\xb9!\x03yeڃ\x854\x1d\xef(\x0f\xcfK\xee\xae\x1c*\xa7\xb8\xaf\x02\x9b\x98\xbaǏ\x9b/0\"ɕ\x1aZl\xe7z\xc2\xcbX\x1ffS\x99\x06}ޗڔc\xa2\xa9\x9dU\x86\xd2\x0f\xa9\x15\x1a\x82\x10\xab^Q\x18{\x9dK7\r\xbbNR\x04\x15Bt\xb5 \xac\xa7\x0ew\x06֢G\xbd\x16\x01\xbfq\xad\xb8*\xa1\xe0\"\\U\xadC\x81\x9d:gz\x0f\x16Fy&j^\x01\x128\xe1[\xa4\xa9u\x82\xe5Kr\xe2\xe3_:q,X\xdfcٖ\xac9a\x00\x92\xf5\xe8\x87i\xa1.a\x80\xd9F\x9fE2\xf67\xd3\xc0\xbc\xb2\xa0\xb0\xd8\x1db:=\x9a?4\xb1\x9f?\xa0\x80\xdf\x13\xe6{\xdb^\\_[C<\x17\x17\x9d\x9e\xac\x8e=n\x8cp\xa1\xb3o\xf8\xde\x11\xf6\x7f:\xf4\xf9\x1a\xbe\xe8:\xde滫\xef\x82c\xd4g\xcf}D\xbeA\xf0|\xa6\x83\xc3UQ\xae\xc04x^\x95\xe8zs\xf7\x1e\nϸ\xbf\xa3Hw\xa6\xb1o\xa4\xb8w\x9c\xf5;#\x03\xe3\x97\xde\x10o\xf74\xbfBƞ\xe6-\xf9\xeeD\xf8\x14+\xf4\x06\t\xc3^\xa9_\x14u\xb3\x11\x01^:%\xbb\xb41\r\x04_\x02!X\xa9\xe6$\xf5\n\xf8\xac#\xca\xe3\xccP\x16iXg\xcc\f\xfe\xc4|F\xfd\xce\x1dP\f\x8at\x95\x82\x92\xa0\x18ޡ\xa1\xc9\x7f\xa4ZF\xef\xd3\x15\x95\xad\xfc2\x99n\xb8VDG\xe5\xf9\xeb\xf1\xfe\r%\xbd\xdd{\xa6\x17\xb7P&\xa3q\x1e\x8b\xa0Z~A\xf1\x1akiҸS2\xf2w\xfc\xc2;&j\xb6\xa2\xf8թ<\x80o@\xfc\xb8ŝ\x8f&\xdf\xf3\xd37l\n\x88\x81\x9f[ \x85\x99\xc1X!Ԩ\x91\xb0\x86\xea5\xdf\\\xaf\x81\xb0?\xc5\xddX\xdf\vZ\x01\xdf\xff\x05\xa9\x9962QkQi\\\x01\xf9x\xae\xcbf\x13w\x9d\b3cx\x94\xf3\x03\xfb\xcc5\xc6n\x18/v\x06\x9c\xbd_\n\xf8\x8c/3\xd6\ao%\x86\x80\xa7ct6\x93\xd9!81\x06~\xa4\xd5\a,\r\x7f\x19\x06\xcb\xff\x01\x00\x00\xff\xffx\xae@\xbaJ\x0e\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4Z_s\x1b\xb7\x11\x7fק\xd8Q\x1e\x92\xcc\xf8\xc8\xdam3\x1d\xbe\xd9r\xd3Q\x9b\xa8\x1aS\xf6K&\x0f\xcbÒ\x87\xe8\x0e@\x01\x1ce6\xcdw\xef,p \xef\x8e )҉}/6\xf1g\xf1\xc3\xfe߅\x8a\xa2\xb8B#?\x90uR\xab\x19\xa0\x91\xf4ѓ\xe2_n\xf2\xf877\x91z\xba~y\xf5(\x95\x98\xc1M\xeb\xbcnޑӭ-\xe9--\xa5\x92^juՐG\x81\x1egW\x00\xa8\x94\xf6\xc8Î\x7f\x02\x94Zy\xab\xeb\x9al\xb1\"5yl\x17\xb4he-\xc8\x06\xe2\xe9\xe8\xf5\x9f&/\xbf\x9b\xfc\xf5\n@aC30Z\xacu\xdd6\xb4\xc0\xf2\xb15n\xb2\xa6\x9a\xac\x9eH}\xe5\f\x95L{eukf\xb0\x9b\x88{\xbbs#\xe6{->\x042o\x02\x990SK\xe7\xff\x95\x9b\xfdA:\x1fV\x98\xba\xb5X\xef\x83\b\x93N\xaaU[\xa3ݛ\xbe\x02p\xa564\x83;\x86a\xb0$q\x05\xd0]1\xc0*\x00\x85\bL\xc3\xfa\xdeJ\xe5\xc9\xde0\x85Ĭ\x02\x04\xb9\xd2J\xe3\x03S\ued40\b\x10\"Bp\x1e}\xeb\xc0\xb5e\x05\xe8\xe0\x8e\x9e\xa6\xb7\xea\xde\xea\x95%\x17\xe1\x01\xfcⴺG_\xcd`\x12\x97OL\x85\x8e\xba\xd9\xc8\xdey\x98\xe8\x86\xfc\x86A;o\xa5Z\xe5`<Ȇ\xe0\xa9\"\x05\xbe\x92\x0e\xe2m\xe1\t\x1dñ>\xdc2\x7fp\x98\xe7\xed\xcecc\x06\bn,\xe1nk\x84 \xd0S\x0e\xc0\x96\x9f\xa0\x97\xe0+b\xce\a\xc5B\xa9\xa4Z\x85\xa1(\t\xf0\x1a\x16\x14 \x92\x80\xd6d\x90\x19*'F\x8b\x89JD\a\xb0\xeeF\xa3\xa7x\xc3\xeb\x7foT\x03@\xf7Z\\\x00\xe5\xacs\xe3\xe2\xc1\xa9\x1f\xfaC'\xf5\xa3\xa2\xb0&\x1dޚZ\xa3 \xcb\xc7W\xa8DM,Y\x04oQ\xb9%\xd9\x030Ҷ\x87\x8d\x19\x82y\x9f\xe8\xf5f\xceaFg;s\xaf-\xae\b~\xd0epP\xacҖ\x06:\xed*\xdd\xd6\x02\x16\xe9\x14\x00\xe7\xb5\xcd*8#\x8e\xbb:\xba\x89\xec\xc8Άg\x1eFߣ\x9d\xfc\xe9\xa4d\x1b\x91Z\xe5-\xe8\xf5\x8a\xf2\xd6\x13\xa7\xd7/\xa3\xbb*+jp֭Ԇ\xd4\xeb\xfb\xdb\x0f\x7f\x9e\x0f\x86\x01\x8cՆ\xac\x97\xc9}Ư\x17\x1cz\xa30d\xf5\xff\x8a\xc1\x1c\x00\x1f\x10w\x81\xe0(A.\xead\x1c#\xd1a\x8a\xe2\x91\x0e,\x19K\x8eT\x8c\x1b<\x8c\n\xf4\xe2\x17*\xfddDzN\x96\xc9$A\x95Z\xad\xc9z\xb0Tꕒ\xff\xdd\xd2v\xac{|h\x8d\x9e\x9c\x87\xe0j\x15ְƺ\xa5\x17\x80J\x8c(7\xb8\x01K|&\xb4\xaaG/lpc\x1c?jK \xd5RϠ\xf2\u07b8\xd9t\xba\x92>\x85\xccR7M\xab\xa4\xdfLC\xf4\x93\x8b\xd6k릂\xd6TO\x9d\\\x15h\xcbJz*}ki\x8aF\x16\xe1\"*\x84\xcdI#\xbe\xb2]\x90u\x83c\xf7\xb4&~!ҝ!\x1e\x8e} \x1d`G*^q'\x85\xe4\xbb\xde\xfd}\xfe\x00\tI\x94T\x14\xcan\xe9\x1e_\x92|\x98\x9bR-\xd9\a\xf0\xbe\xa5\xd5M\xa0IJ\x18-\x95\x0f?\xcaZ\x92\xf2\xe0\xdaE#=\xab\xc1\x7fZr\x9eE7&{\x13\xd2\n\xf6e\xada5\x17\xe3\x05\xb7\nn\xb0\xa1\xfa\x06\x1d}fY\xb1T\\\xc1Bx\x96\xb4\xfa\xc9\xd2xqdoo\"\xa5:\aD;\xca_\xe6\x86J\x16,\xf3\x96wʥ\xec<\xddR[\xc0\xf1\xf2!\x9f\xf2\x0e\x80\xbf\xac\x97\x1b/:\xa5t\xfc\xbd\xc9\x11J\x80U\xcfa'o\xdc9\xcfz\xe8<\xfb_r\xe1\xdb=\x96\x8cv\xd2k\xbba\xc2\xd1{\x8f\x15\xe2\xa0l\xf8+Q\x95T_r\xbd\x9b\xb0\x13\xa4\x12\xccv\xda*4\xbb\xa2H5\x00\xd5j\xa5\xd9\xc4\xc6Ҁ[\xcf\xcbX\xc9\x1d\xf9\xfc]U\xa00\xda\xc9\x17\x95\nvy \xf4\xf3\xbd\xf1\xa5\x17Zׄc^*-\xe8ĝﴠ\x9c\xb0x+\xf8\n}\xc2Ƌl\xab\xd4>o\xf9\xd3\xea,q\x18-N\xe0\xeaND\xb0\xb4$K\xaa\xa4\xe4\xfb\x8f\xe5c\x19d\xfdLi\x1f\xe3a\xfb\x80#\x812\x8b\xf8\xf5\xfdm\n\x86\x89\x89\x1d\xf6\xbdxw\x92?\xfc-%\xd5\"\xe4\x0e\xa7\xcf\xcej.\x7f\xb7\xcb\b\"D\x04\xaf\x01\xc1H\x8a\x19\xf76\x1a\x83T\xce\x13\x8an\x90\x9d\xa0\xa5n\xeeE\xf4\xf4\aA\xf2\xb7\x8b\xda,\x13@\x8e\xb2%\xc7+\xb4\x04\xb5|\xcc\xd8O\xfc\xaeC\xae\xb8\x83\xf9+[\xcfo\xd7\xf0Mt^\xd7\xfc\xf3:\xc2ئ-}\x03\xdb\xc1\x89Vf\xe5jE\xbb\xa4tOY8\xccr\x80\xfa\x16\xb4\xe5\xbb*\xdd#\x11\b\xb3\x9cb| \xb1\a\xef\xa7W?_\xc37C\x1e\x1c8J*A\x1f\xe1\x15{\x9f\xc0\x1b\xa3ŷ\x13x\bz\xb0Q\x1e?\xf2Ie\xa5\x1d)Ъ\xdeĂ`M\xe04W\x94T\xd7EL\x10\x05<\xe1\x06\xf4\xf2\xc09ID\xac\x9a\b\x06\xad?\x9a$v|8n4\xfbYS\xfa\x9eg/!\x8bz\x96\xf5~\xb1\f䙜\b\xe5\xc2'p\xa2_j]\xc0\x89\xc7vAV\x91\xa7\xc0\f\xa1K\xc7|(\xc9x7\xd5k\xb2kIO\xd3'm\x1f\xa5Z\x15\xac\x8cE\x94\xba\x9b\x86\n~\xfaU\xf8\xe7ҋ\x87Z\xffSo?\xe8M|~\x16\xf0\xe9nz\t\aRv\xff\xfc\xd8u\x90\x0f\xf3.\xe1\x1c\xd3d\x9b\x7f\xaadY\xa5Z\xaf\xe7m\x1b\x14\xd1\x1d\xa3\xda|!\xdba>\xb7\x96\x11m\x8a\xaeUY\xa0\x12\xfc\x7f'\x9d\xe7\xf1K\x18\xdb\xcaOr.\xefo\xdf~I\x8bj\xe5%\x9e\xe4@\r\x13\xbf\x8f\xc5\x0eUѠ)\xe2j\xf4\xba\x91\xe5h5\xe7\U00037085\xb4\x94dO\xa4\x7f\xef\x06\x8bS\x82\x9a\xa9\x06\xb6k\xce\xca?=\xae2\t_\xbf\x8b{,-<ʯӪ\xf0\x80+\ah\t\x10\x1a4\xac\x11\x8f\xb4)b\xc6aPr\xba\xc0\x19\xc1\xb6k\x05hL\xcd1=f\x11\x19\x8a]\xfe۱\a]\xb8\xdf!\x86dE\x99\xbats\xf2^\xaa/Ȝ\xf7# \xbf/\xa3\xb6=\xccR\xab\xa5\\\xb56T\xa0\xfb\x9cRm]㢦\x19x\xdb\x1e\xaa\xb9\x8e2\xf2\x81\x97\x1c\xbf\xff\xfb\xdeҤ\xe1'\x1a\xae\xf9[\rڰ\xfb\x97!\xd56\xfbP\nx\xd4Fbfܒ\xf3{\xd6\xcb\x13\xd7\xd7\xe7\xd8XT\xcaKJ\xee\xeeq$S\x95v\x8a\xde%\xf0\xa92\xed\xf7óB?\xc37pu\xcf\xe5\xc8\x10w\x91o\x97\x8c\xd6p\xcd<\x1a2Z\x8cF\x86np49\xe8\xd9\xf7\x91\xee\xf7\x90\xc2S\xcc\x19]\xa4\xf8\xc4\xd4\xf14\x06G\x9f\x1e\x9e8\xed\xbe\xb4\x8fTj.\xbf\x06\xfd\xec\x8b\xda,\xfbdB\xff\u05ca\xce0d\xc3~\xa0\xf7J\xd5\x1d\x9ck\x04\xf5\xc9ŝ!Gaj$B\x19\xc5U\xde\x12eM\x02\xd2S\xe4\x99T\x16\xb4\xe4\x18\x1d\x8d45\":x\x87\v\x98\x87\x8a\xc0\x85n\xea\xd7nK\xb3u$B3/Ä\xfd\x88\xbdԶA\x1f\x1f\x06\n&q\x99\xf7\xca\xdalC\xce\xe1\xea\x94\xd1\xfe\x18W\xc5\xfeL\xb7\x05p\xa1[\xbfm\xd0\f\"\xd2\u05eeS\xb4\xf3zD\xd9\xd6\xc7P\xc7\xd1WI\xa5\x97m]\x87=}\xef\xb0{\xa6\x0e\xa8\x16\x94\xcf\xeb\x8e4\x88\x8e\x01\xacНb\xd5=\xaf\xc9Y\xdd֥\x1d5;8\xe2\xbe\xef\xe8)3\xba\xf7nܟ\xbcI&\x93\x99\xfb>X\xc3Y\xf7\xef\x0e\xba\xc4ܷM\xcdJ\xd7\xc9µ\xc7\x1aT\xdb,\xc82s\x16\x1bOn\xe4\xf8Q\x89>'s\xd5\xdfn\x7f\x12j\xa4\xd4u0\xba^l09\xafAHgj\xdcl\xef\x12rn\xb6\xaf|cz\xa7\xe4\xc9\xd2\r\x1d\xca!\x8e\xb7\x16\x03\xa6\xb7Z\x1d\xa8R\x93\x91K\xe5\xbf\xfbˑ\xa4]*O\xabQ\x18\xe9晝o\xf8\x94?\xe6\x84#9\x90Sh\\\xa5\xfd\xed\xdb\x13\xaa1\xdf.L&\xb2\xcb\xe7\x83C\fo\x1eݢN\x152Pw\x0e\xe7,\xfb\x1d\xfe\x19\xc3%Z<\x1fP8\x11\xaf\xba\xbf\xaa\xc8E\x859\x19\xb4\xec\x13\u008b\xda\xcd\xf8}\xf8\x058\x19\x1a\xe0\x9c\xed\xc6\xf4\xb7\xacP\xad\xb2\xfd\x11\xadB\x02\xa7\xed\xfe\xf3&\x9c\f@\xc3\v}\xceؓU\xa7\xbd\xc1\x80\\\xf4hw\x8fI\xfd\x91v\xb1}g\x9d\xc1\xaf\xbf]\xfd?\x00\x00\xff\xff\f\x19\xe2z\x0f%\x00\x00"), - []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4Y_\x93۶\x11\x7fקع<$\x991\xa5\xdam3\x1d\xbd\xd9\xe7\xa6smr\xbd\xb1\xce~\xc9\xe4aE\xacHD$\x80\x00\xa0d5\xcdw\xef,@R\xfc'\xe9tnl\xbe\xdc\t\x00\x17?\xfc\xf6/\x96I\x92\xcc\xd0\xc8\x0fd\x9d\xd4j\th$}\xf4\xa4\xf8\x97\x9bo\xff\xe6\xe6R/v/g[\xa9\xc4\x12n+\xe7u\xf9\x8e\x9c\xaelJoi#\x95\xf4R\xabYI\x1e\x05z\\\xce\x00P)푇\x1d\xff\x04H\xb5\xf2V\x17\x05\xd9$#5\xdfVkZW\xb2\x10d\x83\xf0f\xebݟ\xe6/\xbf\x9b\xffu\x06\xa0\xb0\xa4%\x18-v\xba\xa8J\xb2伶\xe4\xe6;*\xc8\xea\xb9\xd43g(e\xe1\x99ՕY\xc2q\"\xbe\\o\x1cA?h\xf1!\xc8y\x17儩B:\xff\xaf\xc9\xe9\x1f\xa4\xf3a\x89)*\x8b\xc5\x04\x8e0\xeb\xa4ʪ\x02\xedx~\x06\xe0Rmh\t\xf7\f\xc5`Jb\x06P\x9f3@K\x00\x85\b\xcca\xf1`\xa5\xf2doYD\xc3X\x02\x82\\j\xa5\xf1\x81\x99V\x0e\xe8\r\xf8\x9cx\xcb\xc0*J%U\x16\x86\"\x04\xf0\x1a\xd6\x045\x12\x11\x84\x01\xfc\xe2\xb4z@\x9f/a\xce\xc4͍\x16s\xd5Ȭ\xd7D\xce\xef\a\xa3\xfe\xc0\xe7p\xdeJ\x95\x9dB\xf6\x7f\x06\xd5\xc3\xf3\xa0\xc5\x13\x91<\xe6\x14\xd64h*Sh\x14dy\xf3\x1c\x95(\b\xd8@\xc1[TnC\xf6\x04\x8a\xe6\xb5ǃ\xe9#y\xdf\xc8\xeb\xcc\\\xc3\xce5Tĵ\xbd\xed?t\x87.\xed\xfb\xa0E\xfd\x02\xd4F\rΣ\xaf\x1c\xb8*\xcd\x01\x1d\xdc\xd3~q\xa7\x1e\xac\xce,97\x01#,\x9f\x9b\x1c]\x1f\xc7*L\xfc\xb186ږ\xe8\x97 \x95\xff\xee/\xa7\xb1\xd5/ͽ\xf6X\xbc9xr=\xa4\x8f\xc3ሖ\x9d-\xab\xd5\xffE\xe0\xae\x19\xd2[\xad\xfa\xbc\xbe\x19\x8cN\x81\xed\bm\xe2\xed<\xb5\x14B\xed\xa3,\xc9y,MO\xea\xeb\xac/O\xa0\x8f\x03qz\xf72\x86\xb24\xa7\x12\x97\xf5JmH\xbd~\xb8\xfb\xf0\xe7Uo\x18\xc0Xm\xc8z\xd9D\xd7\xf8t\x92Gg\x14\xfa\xcc\xfe7\xe9\xcd\x01\xf0\x06\xf1-\x10\x9cE\xc8E'\x89c$jL\xd1y\xa4\x03Kƒ#\x15\xf3\n\x0f\xa3\x02\xbd\xfe\x85R?\x1f\x88^\x91e1\xe0r]\x15!\"\xed\xc8z\xb0\x94\xeaL\xc9\xff\xb4\xb2\x1d\xfb\"oZ\xa0'\xe7\x03\xd7Va\x01;,*z\x01\xa8\xc4@r\x89\a\xb0\xc4{B\xa5:\xf2\xc2\vn\x88\xe3G\xb6\x1f\xa96z\t\xb9\xf7\xc6-\x17\x8bL\xfa&\xa5\xa6\xba,+%\xfda\x11\xb2\xa3\\W^[\xb7\x10\xb4\xa3b\xe1d\x96\xa0Ms\xe9)\xf5\x95\xa5\x05\x1a\x99\x84\x83\xa8\x90V\xe7\xa5\xf8\xca\xd6I\xd8\xf5\xb6\x1dyd|B\"\xbcB=\x9c\x19A:\xc0ZT<\xe2Q\vM|\x7f\xf7\xf7\xd5#4H\xa2\xa6\xa2R\x8eKG\xbc4\xfaa6\xa5\xdap\x84\xe6\xf76V\x97A&)a\xb4T>\xfcH\vIʃ\xab֥\xf4l\x06\xbfV\xe4<\xabn(\xf66\x94\x1d\x1c\\+\xc3f.\x86\v\xee\x14\xdcbI\xc5-:\xfa̺b\xad\xb8\x84\x95\xf0$mu\x8b\xa9\xe1\xe2Hog\xa2\xa9\x84N\xa8vXݬ\f\xa5\xacY&\x97_\x95\x1b\x99F\x9f\xdah\v8Z\xdfgj:\x04\xf0\xb3\xc6t[\x99\x95\xd7\x163\xfaAG\x99\xc3E\x97̎\x9f7S\x82\x1aĪ\x93P\xe3\x8e\xe0\xe2J(\xea\xa5\x13\"\xf79Y\xea\xbec\xc9h'\xbd\xb6\a\x16\x1cS\xf1\xd0$Nj'\xf0\xa0Ņ\xb3q.\t\x0ediC\x96TJM\xb89W&M\x80\xefT\vc\x88\xa7\xf5\x01gB\xf3$\xe0\xd7\x0fwM\xf8m\x18\xae\xa1\x8f\"\xecEz\xf8\xd9H*D\xc8V\x97\xf7\x9e4\x04~\xee6\x11D\x88A^\x03\x82\x91\x14\xcb\xe06\xfe\x83T\xce\x13\x8az\x90\xdd\xceR=\xf7\"Ɩ\x93 \xf99\xe6\tV\t \xc7:)\xe0\x9f\xab\x7f\xdf/\xfe\xa1\xe39\x00Ӕ\x9c\v\xe5\x00\x95\xa4\xfc\x8b\xb6$\x10\xe4\xa4%\xc1u\x11\xcdKTrC\xce\xcfkid\xddO\xaf~\x9e\xe6\x0f\xe0{m\x81>bi\nz\x012rކ\xcf\xc6j\xa4\x8b\ao%\xc2^\xfa<\x005Z\xd4\a܇#x\xdc\x12\xe8\xfa\b\x15A!\xb74\xcd>\xc0M(4\x8f0\x7fc\xd7\xfa\xfd\x06\xbe\x89\xcer\xc3?o\"\x8c6Qv\xbd\xef\b\xc7\xe7\xe8\xc1[\x99et\xachG\xc6\u0081\x9dCⷠ-\x9fU鎈 \x98\xf5\x14\x03\x12\x89\x11\xbc\x9f^\xfd|\x03\xdf\xf498\xb1\x95T\x82>\xc2+\x90*rc\xb4\xf8v\x0e\x8f\xc1\x0e\x0e\xca\xe3G\xde)͵#\x05Z\x15\x87xA\xd8\x118]\x12\xec\xa9(\x92X\x92\b\xd8\xe3\x01\xf4\xe6\xc4>\x8d\x8a\xd84\x11\fZ\x7f\xb6,\xa9y8\xef4\xe3<\xdd?\x05\xbc\xbb[<\x87\x81\xa6\x9e|z\xee:\xc9ê\xaep\x862\xd9\xe7\xf7\xb9L\xf3\xe6vщ\xb6%\x8a\x18\x8eQ\x1d\xbe\x90\xef0ϕeD\x87\xa4n\x9e%\xa8\x04\xff\xef\xa4\xf3<\xfe\x1cb+\xf9I\xc1\xe5\xfd\xdd\xdb/\xe9Q\x95|N$9Q5\xc7\xe7crD\x95\x94h\x92\xb8\x1a\xbd.e:X\xcd5\xe3\x9d`%m$\xd9\v\xd5\u07fb\xde\xe2\xa6z\x9d\xa8>\xdb5W\x95\x9fN\xa1q\xb9\xf6wo/\xe0X\xb5\v\x1b\fG\x1d\xd6Eg#kЙ\xba\x0eO\xf0\xad\xfbӑ\xab\x0f\xaa\xbf\xbaA\xa6\xad\xcc$߿\xdb\xf0\x11\xae$\nK\xecv$\xbbO\x89\xc6H\x95]\x85\xb5i\xf0\xad\xc8\xf35v\xa2p\xee\xb6fϕ\xd7g\xed\xee\xb2K\xbd\x1f\x00\x01\xb4\x04\xc8gb\rm\xe9\x90\xc4*Π\xe4\x12\x8c\xab\xac\xbaT]\x13\xa01\x05\xd7I\xb12\x9b\xf2\xf5\xa6]\x99j\xb5\x91Ye\xc3\xe5h̔\xaa\x8a\x02\xd7\x05-\xc1\xdbj,\xe8\x8c\xfbt;\xa5\x174\xfe\xbe\xb3\xb4Q\xf7\x85^\xed\xf4\xa9z\x1d\xdc\xf1aHU\xe5\x18J\x02[m$N\x8c\xb3\xb1\x8f\x1c\x9d'nn\xae1\xa9\xe8I\x178\xa8\x1b\x8b\x13\x17\xd9\xda\x11벞G\xf8\xf2\x18\xdcq:;^렖~\xad\xf8\x8e\xd2G\x98L\xdf\xd9\ak\x8c\x16\xb3!i\xdd\xd86\x980\xf5\x9f\xf1ע\xc1|\xfb-\xec\x8f\xd9\xe1L\x99\xe05\x1btT\xcfF\x8a\x17a\xa2\x1e\xf2\a\xc6켕j\x9dB\xf1N\x96\x04\xa2\xb2A\xb5\xbc\xff\x9c\xc0o\xa4\x1b\xc3ۣc\x88և\x8d\xa7\xc1\x84y\x16\xe9<\x96f\x88\xaa\xf3j\x84%\xd0S\nԽ.MA\x9e\x04,\x0f\x9e\x9a\xad\xac\xb4-\xd1\xcfA*\xff\xfd_N\xf3Q\x136\r\xaf\xbeѪO\xcek\x1e\x85\xcepD\xc2\xdaZ\x93M2\xa4=\x16\x9f\x02ij\x80ם\xf7#\x92(\xb7;~\x11\n\x9b\x1e\xe8\x15\xf8\r\xc1k̷\x95\x81\x85\xd7\x16\xd7\x04?\xea<\xaap\xbf!Ka\xc52\xae`\x0f\x06ɺ\xd36\xa9:C\xf94\xae\xad\x855\xb2\x06\xfa\xeb\x7f\xe8\xb3\xd8Wn\t\x93\xf6Մ\xa2iX!\xb5J\x1b٫5=\xcb\xc0\xbaD*-\xa8\xc3\xda\b\x97t`\xac\xceɹ3\x86\xcfBzH\x1e\x8f\x03\x17)\xdaPX\xd3\x00\xaaL\xa1Q\x90\x05\xafa\x83J\x14\x14u\xe8-*\xb7\xaa-c\xac\xc2\xe6\xb5w\aӇ\xf2\xbe\x91י\x19a\x8aKw\xdf\xc50\x98o\xa8\xc4y\xbdV\x1bR\xaf\x9e\x1e>\xfcy\xd1\x1b\x06\xa6Ő\xf5\xb2\x89\xcc\xf1\xe9$\x9e\xce(\xf4\xf7\xfc\xbf\xac7\a\xc0\x1f\x88o\x81\xe0\fD.pQ\xc7W\x125\xa6ȑt`\xc9Xr\xa4bN\xe2aT\xa0\x97\xbfR\xee\xa7\x03\xd1\v\xb2,\x06\xdcFW\x85\xe0ĵ#\xeb\xc1R\xae\xd7J\xfe\xb7\x95\xed\x98p\xfeh\x81\x9e\x9c\x0f\x8eh\x15\x16\xb0â\xa2\x17\x80J\f$\x97x\x00K\xfcM\xa8TG^x\xc1\rq\xfc\x14\xacI\xad\xf4\x1c6\xde\x1b7\x9f\xcd\xd6\xd27\xe98\xd7eY)\xe9\x0f\xb3\x90Y\xe5\xb2\xf2ں\x99\xa0\x1d\x153'\xd7\x19\xda|#=徲4C#\xb3\xb0\x11\x15R\xf2\xb4\x14_\xd9:\x81\xbb\xdegG\x8a\x8eOH\xa2W\xa8\x87\xb3*{\x02֢\xe2\x16\x8fZ\xe0!\xa6\xee\xed\xdf\x17\xef\xa0A\x125\x15\x95r\\:\xe2\xa5\xd1\x0f\xb3)Պ\r\x9f\xdf[Y]\x06\x99\xa4\x84\xd1R\xf9\xf0#/$)\x0f\xaeZ\x96ҳ\x19\xfc\xa7\"\xe7YuC\xb1\xf7\xa1d\x81%;\x14\xc7\x011\\\xf0\xa0\xe0\x1eK*\xee\xd1\xd1\x1f\xac+֊\xcbX\t\xcf\xd2V\xb7\x10\x1b.\x8e\xf4v&\x9a*\xea\x84j\x87\xf1ma(g\xcd2\xb9\xfc\xaa\\\xc9:\x93\xac\xb4\x05\x1c\xad\xef3\x95\x0e\x01\xfc$3\xcap\xd1%\xb3\xe3\xe7uJP\x83Xu\x02y\x9d\xef\\\x9d\xa8\x8a~\xa2\xea>\xa3\x1ci\xc9h'\xbd\xb6\x87c\xa6\x1c\x9a\xc4I\xed\xf0\x93\xa3ʩ\xb8e{\xf7\xe1M\x90J0\xefԚ4\a\xa3(5\x00\xd5j\xad\xd9\xc9F\xea\x80\a\xcf\xeb\xd8\xce\x1d\xf9\xf4f\xd5\xc9\xcc&\x15\x1ckL\xe8֒\xc3m/\xb5.\b\x87l\x1a-.l\xfaIׁ\xc3Ҋ,\x85\xfc\x1fì\xd1!\x18{\x94\xaa\t\x1f\xb1\xe4\x06\xaf\x13\xfbXr\xb89\xa5\x9a\xd3v\bgRR\x12𫧇&\xed4\x96UC\x1fe\x96.?I\xb3\xe0g%\xa9\x10!Q_\xfev\xd2B\xf8yXE\x10!\xf6z\r\bFRN\xbd\xbc\aR9O(\xeaA\x0e7\x96\xea\xb9\x171\xa6\x9e\x04\xc9\xcf1?\xb2J\x009\xc6K\x01\xff\\\xfc\xfbq\xf6\x0f\x1d\xf7\x01\x98s%\x14\xce*T\x92\xf2/\xda\xf3\x8a '-\t>}дD%W\xe4\xfc\xb4\x96F\xd6\xfd\xfc\xf2\x974\x7f\x00?h\v\xf4\x11\xb9\xe8\x7f\x012rަ\x8d\xc6j\xa4\x8b\x1bo%\xc2^\xfaM\x00j\xb4\xa87\xb8\x0f[\xf0\xb8e\x8f\x89[\xa8\b\n\xb9\xa54\xfb\x00w\xa1x:\xc2\xfc\x8dC\xca\xefw\xf0M\f\x12w\xfc\xf3.\xc2h\v\x84n\xd49\xc2\xf1\x1b\xf4\xe0\xad\\\xaf\xe9Xh\x8f\x8c\x85\x13\x1a\xa7\x82oA[ޫ\xd2\x1d\x11A0\xeb)\x06b\x12#x?\xbf\xfc\xe5\x0e\xbe\xe9sp\xe2SR\t\xfa\b/\xd9\xc7\x037F\x8bo\xa7\xf0.\xd8\xc1Ay\xfc\xc8_\xca7ڑ\x02\xad\x8aC\xac7w\x04N\x97\x04{*\x8a,\x96b\x02\xf6x\x00\xbd:\xf1\x9dFEl\x9a\b\x06\xad?[\x8e\xd5<\x9cw\x9aq}\xd2<\xcf\xf3\x97P\xaf<\xcb{\xbfX\xae\x7f&\x13\xa10\xff\x04&\xbaG\x9d\x1b\x98\xd8VK\xb2\x8a<\x052\x84\xce\x1d\xf3\x90\x93\xf1n\xa6wdw\x92\xf6\xb3\xbd\xb6[\xa9\xd6\x19\x1bc\x16\xb5\xeef\xa1e3\xfb*\xfcs\xeb\xc6C\x9f\xe5Sw\x1f\x84|9\n\xf8\xebnv\v\x03M\x1d\xfd\xfc\xdcu\x92\x87E]\xd9\re\xb2\xcf\xef72\xdf4\xa7\xaaN\xb4-Q\xc4p\x8c\xea\xf0\x85|\x87y\xae,#:du\xc31C%\xf8\xffN:\xcf\xe3\xb7\x10[\xc9O\n.\xef\x1f\xde|I\x8f\xaa\xe4-\x91\xe4\xc4i!>\x1f\xb3#\xaa\xacD\x93\xc5\xd5\xe8u)\xf3\xc1j\xae\x95\x1f\x04+i%\xc9^\xa8\xfe\xde\xf6\x167U{\xa2\xean\xd7\\Uv;\x85\xc6m\xb4\x7fxs\x01Ǣ]\xd8`8\xea\xb0.:\x1bY\xec\x12gk\xcdsx\x82o=\x9e\x8e\\}P\xfd\xd5\r2m\xe5Z*,\x8e\x110\x1c\xc5\x14\x96\x18~%t_\xa21R\xad\xaf\xc2\xda\xf4\x8b\x16\xe4\xf9\xf8\x9e(\x9c\xbb\xed\xecs\xe5\xf5Y\xbb\xbb\xecR\xef\a@\x00-\x01\xf2\x9eXC[:d\xb1\x8a3(\xb9\x04\xe3*\xab.U\x97\x04hL\xc1uR\xac\xccR\xbe\xdet\xbfr\xadVr]w\"\xc7L\xa9\xaa(pY\xd0\x1c\xbc\xadN\x1d\x82\x92\xee\xd3m\xbc]\xd0\xf8\xfb\xce\xd2F\xdd\x17Z\x7f\xe9]\xf5\x1a\x82\xe3͐\xaa\xca1\x94\f\xb6\xdaHL\x8c\xb3\xb1\x8f\x1c\x9d'\xee\xee\xae1\xa9\xe8I\x178\x88g\xd0\xd4\x01\xbevĺ\xac\xaf\x8f\xac\xd1\x1d\xd3\xd9\xf1Z\a\xe5\xa35\x9fQ\xfa\b\xb3t\xafb\xb0\xc6h1\x19\x92֍m\x83\xc9cd\x1aN\xf4\x9d~0\x1b)xV\x9b'4\x9e\xafi\xf4\xc4륚\xf7\x98V}s\xe9\xc4\x05\xfbͭ\x1e>\x13\x1aO\xa2\xed\xc9\xdf\xd2\ay5\x14\x12\x1a\xb4V\xd4N\"Kj\x9b\x06\xb5\x9d\xd8c\x1b#\x86lc\xc9`\xd2\" 4\xd9]h4~\xed\xa24\xe9\xa0r$Bl\x1d}|$\xa1\xb9\xf3\x11\xe8)\xe3\xf7o\v \xe9\xe6Q\xbc\xee\xea\xdej\xdc\xd4I\x1a\x8b\x19s\x88-mᾥ\xb9hKQv\x94\xd7\x12\x16ő\bGX>a\xafP\x16$\xa0\xbd̽\x9a\xf9\x04\xe8qq\xf39\xc9/\xc99\\_\nZ?\xc5U\xb1\x93U\xbf\x02\xb8ԕ?a\x95_\xbbڵ\xae\xca\xc9J\x8bKH\x1e\xb5\b0\xd4\xe9+\xac1\x9a\x84Z\xba\xd7ZWa\fM\xc2KM?^\x93\n5-\xe4\xf3\xb1\x06\xce\xe4\xb0G\xda'F\x1b\x0fNL=\xd5a!15\xba_\xefN\xc6\xcel\xaa\xa6i\xe6\x922\xdb\xcb\xeb\xc4\xdc\x0f\xc1]\xaeb\xbb\xc6wK@h\xfb\xba\x1b]41 \\:\xab\xaa\\\x92eU\x84k\xedF'm\x05\x8cJt5\x97:\x9c\xb7\x12\x9a4\x1cE\xd5\xfd\xa5\xba!\x1d\xbc\xdck\x10ҙ\x02\x0f\xedf\u0089\x88]:ݞ?\xfaU\x13\xab8\xf3\x9c\xa8\xdb\xcew~\xdb?\x02H\x9f\xf7R7\xf9\xfdg|'?\x98o/\xf7?\xcf\x17\xceԝ\xfd?\xb6\xb8\xc5@\x16=\t\x97\x92E\xfd\xc7\x1f\xd7\xc7\xf8\xfeg\xfe\xc8\xf0\x9edo4\x18\x90\x8b\x8e\xec\xfa\n\xa9;R-\xdb\xfb\xd59\xfc\xf6\xfb\xe4\xff\x01\x00\x00\xff\xff\xec\xe3\xc3\ac%\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xdc=[s\xdb8w\xef\xf9\x15\x98\xf4a\xdb\x19\xcbi\xa6\x97\xe9\xf8\xcd\xf5:\x8d\xfb}\xebx\xec4\xfb\f\x91G\">\x83\x00\x17\x00\xa5h\xdb\xfe\xf7\x0e\x0e.$%\x90\x84d˛-^2\xa6\x80\x03\xe0\xdc\xcf\xc1\x01\xb2X,\xdeц}\x03\xa5\x99\x14W\x846\f\xbe\x1b\x10\xf6/}\xf9\xfco\xfa\x92\xc9\x0f\x9b\x8f\uf799(\xaf\xc8M\xab\x8d\xac\x1fA\xcbV\x15\xf03\xac\x98`\x86I\xf1\xae\x06CKj\xe8\xd5;B\xa8\x10\xd2P\xfbY\xdb?\t)\xa40Jr\x0ej\xb1\x06q\xf9\xdc.a\xd92^\x82B\xe0a\xea\xcd?^~\xfc\xd7\xcb\x7fyG\x88\xa05\\\x11\x05\xdaH\x05\xfar\x03\x1c\x94\xbcd\xf2\x9dn\xa0\xb00\xd7J\xb6\xcd\x15\xe9~pc\xfc|n\xad\x8fn8~\xe1L\x9b\xbf\xf4\xbf\xfe\x95i\x83\xbf4\xbcU\x94w\x93\xe1G\xcdĺ\xe5T\xc5\xcf\xef\bхl\xe0\x8a\xdc\xdbi\x1aZ@\xf9\x8e\x10\xbft\x9cv\xe1W\xbd\xf9\xe8@\x14\x15\xd4ԭ\x87\x10ـ\xb8~\xb8\xfb\xf6OO\x83τ\x94\xa0\v\xc5\x1a\x83\b\xf8\x9fE\xfcN\xc2B\tӄ\x92o\xb8Q\xbb\x1aD<1\x155DA\xa3@\x830\x9a\x98\n\bm\x1a\xce\n\xc4;\x91\xab\x1e\xa40J\x93\x95\x92u\amI\x8b\xe7\xb6!F\x12J\fUk0\xe4/\xed\x12\x94\x00\x03\x9a\x14\xbc\xd5\x06\xd4e\x04\xd4(ـ2,`ٵ\x1e\xef\xf4\xbeNm\xcc6\x8b\v7\x8a\x94\x96\x89\xc0m\xc1\xe3\x13J\x8f>\"W\xc4TLw[\r\xdb#T\x10\xb9\xfc\x1b\x14\xe6r\x0f\xf4\x13(\v\x86\xe8J\xb6\xbc\xb4\xbc\xb7\x01e\x91Uȵ`\xbfG\xd8\xdan\xdcNʩ\x01m\b\x13\x06\x94\xa0\x9cl(o\xe1\x82PQ\xeeA\xae\xe9\x8e(\xb0s\x92V\xf4\xe0\xe1\x00\xbd\xbf\x8e_\x90xb%\xafHeL\xa3\xaf>|X3\x13$\xaa\x90u\xdd\nfv\x1fP8ز5R\xe9\x0f%l\x80\x7f\xd0l\xbd\xa0\xaa\xa8\x98\x81´\n>І-p#\x02\xa5\xea\xb2.\xff.\x12u0\xad\xd9Y\x1e\xd5F1\xb1\xee\xfd\x80\x02q\x04y\xac\xa88\xc6s\xa0\xdc\x16;*\xd8O\x16u\x8f\xb7O_\xfbLɴ'J\x8f7\xc7\xe8c\xb1\xc9\xc4\n\x94\x1b\x87\xacia\x82(\x1bɄ\xc1?\n\xce@\x18\xa2\xdbe͌e\x83\xdfZЖ\xdf\xe5>\xd8\x1b\xd4:d\t\xa4mJj\xa0\xdc\xefp'\xc8\r\xad\x81\xdfP\roL+K\x15\xbd\xb0DȢV_\x97\xeewv\xe8\xed\xfd\x104\xe2\bi\xbd\x16yj\xa0\x18H\x9a\x1d\xc6VA]\xac\xa4\x1a(\x19;d\x88\xa3\xb4\xf0\xdb洈U\x8b\xfb\xbf\xccq\x99m\xff\x1eG[~\xb3+k\x05\xfb\xad\x05T\xa6N\xfc\xe1P_\xa9\x9ej\x1f6\xcbF\xfb\xd4\x1dE\xb4m\xf0\xbd\xe0m\te\xd4\xeb\a\x1b\xcc\xd9\xc6\xed\x01\x144z\x94\t+D\xd6\xfaؽ\x88\xeeWT\xe0T\x01\x11\xd2$\xe01\xe1\xe0\x11&\x10\x03I\x9a`G\x03ubœ[&D\xb4\x9c\xd3%\x87+bT{\x88F7\x96*Ew#\xd8\n\x1e\xc0\x8b\x90\x15\x81xU\xc3Y\x81$\x8f\n\x05\xf1\xf5\xe7E\x15\xd3VQ\x86]>HΊ\xdd\f\xben\x93\x83\x82\xb4z\xd9\xf5;$K\xa8\xe8\x86I\x95\x12\x03\xa9\xb0kϞwjZZ-\xe9\x81\xec۸\xcc\r'\x91UI\xf9<\xc7\x10\x9fm\x9f\xce:\x90\x02\x1dʸ\x15Omo\xbb\x97@\xe0;\x14\xadI,\x93\x90\xb2E\xd3$\x15i\xa46\xe3t\x1fW]\xa4\xef\x1c\xa5~\x9c`\x9a\x83\x9d%Y\xdd5\xaf\x84\x03Q-\x0e\x06\nY\n\xb0ۨ-Q\xbb\xbeJ\xb6\xae\xef(RȒj(\x89\x14\xa33#\xbb\xb4\x1c\xb4\x9f\xabD\xce\xe8\xf4\xd0E\xb7\x7f\xf4x\b\xa7K\xe0D\x03\x87\xc2Hu\x88\xcc\x1c\x94\xba\x96\xa3XGP\x99ЦC\t\xe860\x01\x92XN\xdfV\xac\xa8\x9c\x87a\xd9\x13\xe1\x90R\x82\xb6\xda\x04]\xe6\xdd\xd8&\xc9\x1c\xf9\xfd$Sڣk3b\xb5\x0f/\xa5Q\xba\x96\xa1\x86\xbb\x96Dm\xa7{\x0ft\x8b\xffn\xe4\xe4\xb6\xff\x7f\"6\x18\x93\x13\x98vB\xfe\t\xba\x9f\xd9<=ʷ\x18ၾ$w+\x02ucv\x17\x84\x99\xf0uN\x12(\xe7\xbd9\xfeĴ9\x9e\xe93I\x93#\x13g\"L\x9c\xe2OH\x174\x19O\xdebd\xd3\xe4\xaf\xfdQ\x17\x84\xad\"\xd2\xcb\v\xb2b܀\xda\xc3\xfeI\xaa>P\xe65\x90\x91c\xf5\b\xe6\tLQ\xdd~\xb7.\x8e\xee\x92`\x99x\xd9\x1f\xec|\xe3\x10A\f\xcd\xf3\f\\\x82\xf12SPc\x1cN\xbe\"6\xbb/\xe8T_\xdf\xff|\x18+\xef\xb7\f\xce;\xd8Ȍйv\xbd\xb7\xa3\xfe\xfa|T\x10~A\x1f(\x06U.\xe7rA(y\x86\x9ds]\xa8 \x96>4tΘ^\x01&\x7f\x90Ϟa\x87`\xd2ٜÖ\xcb\r\xae=C\xc2\xf5O\xb5\x01\x0e\xed\x9a|X\xec\xf0d? \"0\x86\xcfe\x03\u05fc($r'閩KB\v\xb8?a\x9bY\xacҟ\xa3\x9f\xfaD\x0e\xf8I;ZZ\x89\xa9\x98\xcfij@\x99\xc9%\xa8k\xdf(ge\x9c\xc8\xc9ȝ\xb8 \xf7\xd2\xd8\x7f0@\xd3\xc8(?K\xd0\xf7\xd2\xe0\x97\xb3`\xd4-\xfc\x9c\xf8t3\xa0\xa0\t\xa7\xe5-\xc2\xfa9?g\xd3,\xb7E\xdc3M\ue10dW\x1cJ2\xa7\xc2\xf4\xae\x9b\xceMT\xb7\x1a\xd3uB\x8a\x05\xda\xcc\xe4L\x1e\xdfR\r\xd0\xfd\xe2I\xfd\x84_\xad\xb1p\xbf\xb8$3\xa7\x05\x94!\xb2\xc4\xec'5\xb0fE\xe6|5\xa85\x90ƪ\xf0<\x8e\xc8T\xac~7DZO\x9e\xf5\xee\xb7\xef\x8b\xe7\x98/XX\x93\xb3\xf0\x10\x8c\xac3p\xe0uw9\xbf\x9f\x85\x95ٌ^\x81\x13f\xbb\x8e$Gǻ\xe6 \xe5\x05\xe8@+\x8e.\xce,uiY\xe2\x11\x1a\xe5\x0fGX\x94#x\xe1X\xd5\xd0[\xbb3\xc15m\xacZ\xf8okiQ\x9a\xfe\x974\x94)}I\xae\xf1\xa4\x8c\xc3\xe07\x9f\x87\xeb\x81ɘ\xb2\xb1SY\xfe\xd9Pnm\xbfU\xe0\x82\x00w\x9e\x80\\\x1d\xf8E\x17d[I\xed\xcc\xf6\x8a\x01\xc7\xf3\x8a\xf7ϰ{\x7fa\xa7\x9f\x9d\xb2\xafd\xde߉\xf7·8P\x18\xd1ᐂ\xef\xc8{\xfc\xed\xfdK\\\xa9LN\xcd\xec6`њ6y\x1c*\x92\xc9\xfa\xae\r8\xa6\x9f\x9b\xef\x92\xf2\xdeɞ\xdam\x16\x8b6R\x9b\xcf\xe9\xbc\xe1\xc8z\x1e\u0088\xa1g\x9cȱ\xcdF\f>\x8f\x16\xf5\xbdu\"W\x06\x94\xcf%:\x1b\x10\xe2\x8f\x17Ff\xa9S\x99\xfebc2\x90\xc6\xfc\xaeE\xf0\f7\xb9\x83\x9b\x9c%\x1e\xe3\xb0Z\xbc\x1c\xe9\xed\xdf~\xef\xe53\xad\xe4ڿ\xfb\x1bym\x87\xba\x90uM\xf7O5\xb3\x96z\xe3F\x06\x9e\xf6\x80\x1c\xf5պEyε\xc8\x1d\x0f\xe1\xf9喙\x8a\tB\x83\xda\x00\xe5\x19\x8a\x92F\xa6rةVQM\x96\x00\"\xa6\xe8\x7f\x04W\xa2f\xe2\x0e' \x1f\xcf\xe0zDt\x9d\xd3ٽ\x894\x89\x94\x8f\x1f\x9c\xc9jdI\xb6\x15(\x180\xc6a\xde\x1d=U!M/eq\x84C\xda\xc8\xf2'MVLi\xd3_\x82&\xadΥ\xf5\x91\xe4\xb3\xeb\xfe\xcaj\x90\xad9'\x82o\xbbi\x06g\xcd5\xfd\xce\xea\xb6&\xb4\x96\xad3\xe6\x86\xd5\xf1TףwK\x99\x89\xc7V\x98\xbf1Ғ\xa0\xe1`\x80,a\x95>\xefM\xb5B\n\xcdJP\xa1J\xc1\x91\x8dI+\x98+\xcax\x9b:%J\xb5c#`q\xab\xd4I\x01\xf0\x177\xb2\x97w\xac\xe4v\x88\xa0̽\xe3A\x1a\x10\xb6\"\xcc\x10\x10\x85\xc58(\xa7\x92q\n\x8f\fD\r\xcb\xd5sy\n\xdc6\x10m\x9d\x87\x80\x05\n$\x13\x93)\xb7~\xf7O\x94\xf1s\x90\xcdr\xde'\xa9\x1e\x81\x96\xa7\xe4h~\xed\r' t\xab\xf0\xf0\xdf\xe9\x8e-\xe3yk\xb6\x94#\x9c\xb6\xa2\xa8\x00\x95\x90\x18\xea\x06\a\x9e\tm\x80\xe6\xf2\x82\xf5\x8aZ!\x98X\xe7\xd1.;\x11\xda5\x87\ua954\x1c\xe8\xf8)d\xd7,\xae\xdf@\x13\xfd\xdaM\xf3BM\xd4\x11\xc1\x1d\x9b#\x1d\xb2)j\x95\x16\xa1\xc6@\xdd8\x91\x93D\xb5\xa2o]Π\x88\x8e\t\xc3\xfd*^3\xbef\x82e\xd0v@\xd7;\xc1L\xdfy\xb4 \xce\xea<\xda\t\xa2;pJ\x86\xedn\x00\xc0\nh\x88Cp\xed\x91k\x8ep$\x97@hYB\xe9r\x97\xd6\x15\xf1a\x89+|\x1b)nH\xee\xeexO0\x8b\xb2\xa1\r\x82N\xccê\r,Z\xf1,\xe4V,0\x18\xd7G\xeb\x90\x13\xb3T/\x9dޜ\xac\x8c\xe6\xf5K\xbe\x9a\x9e\xd3BC~\xcd\xe7\xa9\xe0?\x9dA\xcbd\xf3\xcdQ\t\x8f).\x98\xd3k\xae\x00{\xe4\xc7\xd9UL\xcd?1\xd8\x1fJ߸b\xe9\x17\x95\xc5ݥA\xf5\x9c\xc2m\x05\xa6\x02\x15J\xb3\x17X\x92^N\x9e\x90v\xc1K\xac\x93\xb3L\x15\\dW\xfe\xb9W9\x87\xd1M\xcb\xf9\x85\xe5m\xda\xf2d8l$\x8a\xd8!geՏ\xa5=\x86\x9c\xea\x8bl<\xf6+-\x86\xf5\x85\xb1\n\"\x14\x18\xca0\xb3\xa7qj\xbfXX\xda;\xdf\x1f\x96S`\xfe/,\xff\x0f/=̨\x94\xc8Gcn\x95fDb\x02V\x82\xc1zh\xec\xea+|?_\xe8\xfbc\xe1\xd4@\xfd\xa5\xf1\x123\xea\xc2f\xa05\x01g\xaf\xde\x04\xadA\xab\x9d+\x10\xed\x80\xcf\x19\xda\xf1ׅ\xbb\x05\x11\xc0\xa4\xf8\xf5k\x05A|}\xf5>\xd3\xe4\x9fI%\xdbDU\xdf\x04\xcaf\xaa;\xe67<(\xf4\xf0\a\n`\xe8\xe6\xe3\xe5\xf0\x17#}\xd9\af\xd1\x12\x800(\xea2\xb3L\x94l\xc3ʖ\xf2 \xb5\xdd\x1d\x02\xc7@\x1d\x9f%\xa0IE\x04\xe3\x8e\x01\xc3\xf8\x01Ñ/\x8d;\x969Z\xc5M\xfb\xa2y\xd5!'ׄ\fk>F\xac\xe1\xb1\xc7\x17\xafR\x05\xfb\x87\xd4z\x1c_\xe1\x91\x13I\xccTs\x9cPÑY,\xf6\xe2\xf3\x96\x9c*\x8dcb\xee\xb3Ud\xbc~\x1dF\x16~\xe6k.\x8e\xc1\xce\xd9\xeb+ް\xaa\xe2mj)2+(^\xaf\x142/\xfa<\xa9\x14`>`\x19\xaf\x82\x98\xad}xQ@sҖfk\x1a\x8e\xa9d\x98\xa5N\x9e\x98\xbdY\xad\u009bU(\xbcm]\xc2$\x17M\xfexL\xe5A\x8c\x93~\xa1M\xc3\xc4\xfa\x90)rYg\x92m\xe6Y\xe6~o!\x03\x9e\xe9\x873]t8\x12\xfa\xba\xeb҉H2\xa4-\x990\xf2\x92\\\x8b\x9d\x87\x9b\x80\xd3\v\x1f\x854\a\x17\xd9첶\x8c\xf3\xfem-\x04;\r\xcaߙԴv\xab\x1a\xf3\xf6\x93t\x95j\xe0\x94\x9f\x148~ك\xd1ώ\xbe\xa5\xe7_\xb7ܰ\x86\x83\xf5\xe86\xacL\xde!3\x15\xec\"\x92\xff&\xf1\x86\xd4r\x87\x90\xbe]\xf4\x9eQ\xec\x9eq\x926\xbf\xc9\x13\xb6\x97Q\xcc~\\\x11{\x06\xcdrE\xf1\r\x8b\xd5߰H\xfd\xad\x8b\xd3g8k\xe6\xe7\xe3\x8a\xd0O>\x81\tG\xfd\xf7\xb2\x84\a\xa9\xcc\\p\xf2\xb0\xdf?q\x92\xda\v\xd8$/\x89\b]\x13\xbb\xc4\x10Ç\x17\xa7m*}\xe8\x19\xdc\xe9_di\xd76w\xc6\xf2\xb8\xd7\xfd\xe0\xae\xf2\n\x14\b\xf7\xcc\xc7\x7f>}\xb9\x8f\xf0S>\xaf\xf7\x8c\xf7\x9e\x97p\x1eL\xe9\x91\xe3\x8f\xe6|1\x93\xc3\x16\xfa\x00\xaf|.B\x1b\xf6\x1f\xf8\xaa\xdb\v\xd2A\xd7\x0fw\b#\xf8i\xf8L\\\xac\xa2\x88'\x96K\xb0\x16+\xa2jT,\xeeV\x03\x88Ê\xdf\xfe3JP\xba'\xb3\x82\xc5d\xa1\xc6\xcb\n\xdeÝ[\xc7\xd8,\x9f\xac\xd3(vD:\x8e\xac\x98*\x17\rUf\x87l\xa3/\x06k\bff*\x9d3\xaaX\x0f\x9f\x01K\xa27\xbc\xfe\x85g\x91\xbbfxڻ\x8f\xbbS\xd61~\xffd\xf6\xe6\xc9+\xaec\xdcb/\x10S\x89\xcf\xc9\x02\x93WK\x93yM\xf4\xf0\xed\xa4\xb4\xcbc\x1c=\xad\xe7l\x14\x1dRM\t0v<\xaa:-h\xa3\xabēK/\xd3u\xf8\x1a\x99\xa1\xa6}\xc9&\x1d\x80\xc1>YQ\xf5\xb4\xd5\x16\x82>\v\xdbFi\xc5a)\xddnm\xb3\xab{a\xfc\xa2\x97)x\x9b#\xe1\xcc\xe7\\N~\xc8šgD\xfd`\xf6˪\xb6CL\x9dp\x18<\xeb\xdae\x14\x19O;\xb1\x99π\xe4\x19\x8c\x13\x9e\xfe@|\xe5\xe2\x8a$_\x04\xc9|\xf5\xe3\x0fE\xf4\x84V\xd3E\x05e\xcb\xe1\xd47\xff\x9ez\xe3\xe7_\xfd\v\xb3e\xbc\xfbg\x91\xdd3\xd0\xd6g\x1e\xbe/\xe8)\xe1!\xf7)9\xe6\xf0ap\xe0\x9e\x17+\xdcK\x94E\x01Z\xafZ\x1e\xaa\x94\n\x05\xd4@\x19\xba3\x1dW|T\x9dM\xdbpIKP7R\xacX\xe2\x84d\x80\xd6\xff\x1at\xde\xe3\xd9\x02?\xb6\xaa{\xdaq\xf2Y\xbc\x17i\xae\x86*\xca9\xf0O\x8c\x83\xfeYn\x85]W\x86@>\xa4\xc6\xf5\xeee\x15\xad\xb2f}GD[/\xad\x93\vƌ\a\x8b+\xa9\xa6+\xa4\x1dޙ0\xb0\x86T|\xbdU\xcc\xc0SC\x95\x06\\Q\xc6\x0e~\xdd\x1b\xe2\xa2\xcf\x15\xa7kW\nW\xb2\x82\x1a\x88\x06\x18g\x18[>\x8e\xd7\b\x8b\xef\xb02I\x8e$\xbd\xb2\x85z\xecJƨX\x8f=/\x9a0\xd5\xc9\aF\x9dE.hc\xf0\x02\f\xd2\x11\x89h<\f|\xb4w\xef\x8d\xd1\x01\xd8qN\xf3e̾`N\x1bZ'\xa2\x84y\xbdss\b\x06\x9f\x05Ve\xaf\xee\xae\xff\xc0b,\xb0#[\xaac1u\xd2\xf7\xee`;0\xe8\xaa[\xd0P\x12\u0600 V\x14)\xe3PNq\xeaWL$\xab\r\xa8\x9ft\x84\x83\x95\x80\x96ş\fU&.\xfdЏYIUSsEJj`aG\x9f溥\x9fIU\xea\xc4\xe3@\xbc\xd9\xe6ţ\b\xd7n\xac\xf5s\xf7\xd1jК\xaeC\x10\xba\x05\x05d\r\xc2\xe2=\xe6\x16\x93\x1eS\xb8\xd2\xe7\x8dE,-\xb5(\xa4\x85i\xa9\x9f\xc0\xb9p\xf1\xf44\xbcO\x8cQ\xeczTE\xa7U\x85\xbf<\xf8\bT\xef?w}\x80\x8bO\xfd\xbe>I\xecv\xec\xceF\xa8+\xf0\xc4\a\x8f\r\x8b\x91uJ\xa6\x8dę\x8f2'\x95\x94\xcfYn\xf6\xe7رK'1\xe1X\t\xafL.ekz~\x8eGxb\x99\xf8\xfc\xe7+\xdb\x17\x84y\xed.P\x8d\xe5V\xf3<\xbd\xcf\x03H1\xbc\x95\x86\xf2`d,_\xc6\x0e\xd5\xc4\x03\x02O\xe1\xf1d\xcew\x17\xfb\x90\xf7^e\xef`W\xddS\x9e^\x13t\xd7\xc7\xc7Ҫ>\xeb\x97\x04\x12_\x01\xed|\x92\xb17\x17\xe7\xec\x1fB\xfd\x84\x8b\xca\xc0\xf1\xe7\xae\xf7\x18\x1e\xdd2\x9d\xc3\f\"\x1di\x12\f>L\x15%ㄥOx\xa9ME\xf5\x9c{\xfa`\xfbD\xb7\xa3g\xae\xa2\x13\xfa8\"\x95\xe9{\xae\vr\x0f\xdb\xc4W\x87,<\xfdB\xa9Jt\xb9\x13\x0fJ\xae\x15\xe8C\xa6[\xe0}F&֟\xa4z\xe0횉/\xe3\x95\xdfS\x9d\x1f\xa82\xcc2\xad[Ob\xecM\xb0q\x89\xdf\xe6G\x8f\xff\xc0\x04\xe5\xec\xf7\x94.\xef\xff87Ä\xbek<\xf2N\xb1P\x01\xf1s\n\xd0k\xe8\x9ft\xcf\xfc\x84y/ɽL\x8a\xb1? fC\xa0L\x93%h\xb3\x80\xd5J*\xe3\xf2\xf7\x8b\x05a\xab\xe0 Y\r\x81q\xa2{͞\xb0T\xe2=\x1e\xbd\x05\x87e\xe5S\x89\n\xad\x0e\x86\x9c5ݹ\x8c$-\n\x1b\x13\xc0\amh*6y\x91\x9e\xc6P\xd5\xcbJ\x8e\n\xb9\xeb\xf7\x8f9\xbe\xa8>\x10\x9cC\x1d^gw\x06\x9d\x8f\x9di\r^\xcb \xdab\xef\x14eB\x9c\x1a\xbb\x1b\x0f\xbb\xf3L\xcd\xd7\beL=\xfa\xfd\r\x1e\xe2\xf6\a\xac\xbe\x93%[QQ\xb1\x1e\xbd\xd0V)ٮ\xab\xc0\x9bc\x0e\x11)[\x8c\x9c\x1bT\x05:\xfc\xc7!\xa6U\xa2wh\xe7k,ƴt\\\uee0f\xf2\x02E\xad\xba\x8b-\x9d\xaa\x9a\xb0\xf9\xd9Y\xc2\x11\x88\xb3\xb6?\x01\x91\xea\x9d(&\xaf\xe0\xf8@\x9bM\xdc՝\xc2P\x12\tQ\x1b\xbf\x1a\x12\"\xc41$\xf4}\x89.\xe2\xf9a02棜\x88\x8ei'\x06\xb78\rj~\xd3}'h\xe8\xee\x1c\x87\x0e=\b\xfeNJ\xbb\r \x1c\x13\xf9\xe2\xdc\xe9\xb8\xf7ǍX7\xd1ۺ=9v\xfd\xb6\ac\xef\n\xa4\x8db\xbbiB\xbc\xf9\xf7l\x95\x92\x17\xf7\xbf3-9\xfc\xc3\xc1\xafo|\x95qK\x95`b}\x12F~\xf5c\x13\xf1\xbc\a{Έ>\xac\xfc\xd5b\xfa\xa4Y:\xf8\x88\f^\xf6\xf0\xecg\xf2_\xfe/\x00\x00\xff\xffP\a\xb5\x16Cm\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec=]s\x1c)\x92\xef\xfa\x15\x84\xeea?B\xdd^\xc7}ą\xde|\xb2gO\xb1\x1e[ai\xf4\xbctU\xb6\x9aQ\x15\xd4\x00\xd5r\xdf\xde\xfe\xf7\x8dL\xa0\xbe\xba\xe8\xa2Z-ygǼت\x86$\xc9L\xf2\x03\x12X,\x16g\xbc\x12\xf7\xa0\x8dP\xf2\x92\xf1J\xc0W\v\x12\xff2\xcb\xc7\xff6K\xa1\xdelߞ=\n\x99_\xb2\xab\xdaXU~\x01\xa3j\x9d\xc1{X\v)\xacP\xf2\xac\x04\xcbsn\xf9\xe5\x19c\\Je9~6\xf8'c\x99\x92V\xab\xa2\x00\xbdx\x00\xb9|\xacW\xb0\xaaE\x91\x83&\xe0\xa1\xebퟖo\xffk\xf9\x9fg\x8cI^\xc2%3\xd9\x06\xf2\xba\x00\xb3\xdcB\x01Z-\x85:3\x15d\b\xf4A\xab\xba\xbad\xed\x0f\xae\x91\xef\xd0!{\xeb\xdbӧB\x18\xfb\x97\xde\xe7\x8f\xc2X\xfa\xa9*j͋N\x7f\xf4\xd5\b\xf9P\x17\\\xb7\xdf\xcf\x183\x99\xaa\xe0\x92}®*\x9eA~Ƙǟ\xba^0\x9e\xe7D\x11^\xdch!-\xe8+U\xd4e\xa0Ă\xe5`2-*K#\xbe\xb5\xdcֆ\xa95\xb3\x1b\xe8\xf6\x83\xe5g\xa3\xe4\r\xb7\x9bK\xb64ToYm\xb8\t\xbf:\x129\x00\xfe\x93\xdd!n\xc6j!\x1f\xc6z{Ǯ\xb4\x92\f\xbeV\x1a\f\xa2\xccrb\xa0|`O\x1b\x90\xcc*\xa6kI\xa8\xfc\x0f\xcf\x1e\xebj\x04\x91\n\xb2\xe5\x00O\x8fI\xff\xe3\x14.w\x1b`\x057\x96YQ\x02\xe3\xbeC\xf6\xc4\r\xe1\xb0V\x9aٍ0\xd34A =l\x1d:\x1f\x87\x9f\x1dB9\xb7\xe0\xd1\xe9\x80\n»\xcc4\x90\xdcމ\x12\x8c\xe5e\x1f\xe6\xbb\aH\x00F$\xaaxmH8\xda\xd67\xddO\x0e\xc0J\xa9\x02\xb8\x80vX4\xb6\nu%\xa0\x80\xe6\f\xddN\x8d\x16FH\xb6\xae\xd1#]2\xd4\x12Q\x19\x11\xd2X\xe0\x11a>\x01\xef\xe0kV\xd49\xe4WEm,\xe8\xdbLU\x90\x87E\xa6Q͜\xca\xc3\x0f\a!\xfb\xf8\xa5\x10\x19 \x1f2WiA\x8b<1\xd1nC\x99]\x05n\xcd\tY\xed\x87\xd0\xc6(\x93\xbaŀņ\xe7\x7f<\xbf \t\xe8\xf7\xde\xef\xc70\xae\xa1!\xd3,\xddL\x16\x7f\xbc\x85\xb0PF\xa8;\xa9\xa3f\xf0\x9dk\xcdw\a\xb8\xde,\xa6\xbd\x00\xdfc\xb0\a\x9c\x97\xa1\xda7\xe2\xfd\xb0\xff\xdf\"\xf7O\xcboC\x8b\xce\\H\xe4s!\x8c\xed\xb1ٸU,$\xebX\b\xe9\t$\x1dLT\x93S\\\xfd'!\xe6I\xe7Nl\xb24\xb2\xe9'\xc0\xbf\x14%7J=\xa6P\xef\x7f\xb1^\xbb\x84\xc52\xda\x18a+\xd8\xf0\xadP\xda\f\x97I\xe1+d\xb5\x8dj\x16nY.\xd6k\xd0\b\x8b\x96\xf9\x9b]\x81C\xc4:\x1c\xbe\xb0\x8eʊV\x18\x8c\xabe:\xb2\x94\xa8\x11\x1b\n\x05\xa8Q\xa8\xce\xc1\xc1Ђ\x1c\x88\\lE^\xf3\x82|\t.37>\xde\xe0\x17\xd3j\x13\x02\xb1\x87\x7fT\xaa]q\x0eM\x18$2\xb1\xb7\xea\xa5$\xa0\x8f_bl\xb4_5N\x89\xb0\x94p\xb0od\xa6\xae\v0\xbe\xbb\x9c\xdc\xe4V']\xb4\xccrk\f\x05_A\xc1\f\x14\x90Y\xa5\xe3\x14J\x91\x03WR\x95n\x84\xb8#Z\xb6\x1fm\xb5\x83\x99\x00\xcb(\xc4݈l\xe3\xdcW\x144\x82\xc5r\x05\x86VExU\x15\x11\xd3ՖI\xe1\xf0\x9dM鍶$h\x90!ܘ.iK\xa2~n\xcb(\xd9۹٧\xfa\xf8:\xff(\xbe\xbf%\xa2\a\xabs\xa4\xb0Oh\x12F\xfb\x05\xc9\xf3!Jz\xa4\xb8\x00\xb3\xec\xac\xce\t\x1b\xbe\xa60\xb4\xe7?\xeem\xa5\xec\x11\xe5\xd7Ż\xe3&\xcc\f\xd6MΩ\x97e\\\xd3Ϳ\b\xdf\xc8d\xddz\x8b5\x8bg\x1f\xbb-/hW\xc03$\xbf`kQX \xa7j\nQ6\x83s\xa7$P\xaa\x05f\xb4Il\xb3͇f\xef(\xa1ŀVC\x00\xceA\x0fQ\x0e\xf1 \x01$k\\\v\xda4\x15\x1aJڌ\xa5H\xb2\xfb\x85\\\xc1w\x9f\xde\xc7c\xcfnI\x94ԽA%LZW\xde\r\x1c\xa3.\xae>T\t\xbf\x90\xbf\xd6\x04\x82n\x13\xfe\x82q\xf6\b;\xe7bqɐo)\x8b\xff|\xf8*\fv,s\xf6^\x81\xf9\xa4,}yQ*\xbbA\xbc\x06\x8d]O4A\xa5\xb3$H\xc4n\U00088ce5(\xa8\r?\x84a\xd7\x12C2G\xa2\x19\xddQ\xae\x90\xeb\xd2uVֆ\xb6Z\xa5\x92\v\xb7,6֛\xe7\x81\xd2=\x16\x9c\xa4c\xdf\xe9\x1d\x1a#\xf7\x8b\xcbZ*x\x06yآ\xa3t\x1an\xe1Ad3\xfa,A?\x00\xab\xd0,\xa4K\xcb\fE\xedG6_\xbc\xd2=\x87n\xf9\xbax\xacW\xa0%X0\v4k\v\x0fŪ2\x91.\xde&\x8c䜌\x95\x05\xce\xf5ĚAZ\x92\xaaG2r\x0eWO%\xd63\xc9D^\x04\xb9]IR\xd0Ml\x9dg\xbdf\xca\xcd1*\xa63\x16\xe7\x02\x94\x9c\xb6\xd6\xfe\x86\x96\x9ef\xe3\xdfYŅ6K\xf6\x8e2{\v\xe8\xfd\xe6\x17&;`\x12\xbb\xadh\x95\xfd\x97Zly\x81\xfe\a\x1a\bɠpވZ\xef\xf9j\x17\xeci\xa3\x8cs\x1b\x9aM\xbb\xf3Gع\x1d\xe5\xa4n\xbb\n\xeb\xfcZ\x9e;_fO\xf14\x8e\x8f\x92Ŏ\x9d\xd3o\xe7\xcfu\xeffH\xf4\x8c\xaa=Q.y\x95.ɔ7;'\xd0\xc0`=8DظI \xc5\x00a\x8a\x02ɢ\\)\x13I\x16\x89\xa0\x95 \xe87\xcaX\xb7\x0e\xd9\xf3\xf7G\x17*UX\x9cd|mA3c\x95\x0e)\x99\xa8\xf8S\x96\xe2\xbb\xe5n\x03\x06\xfc>\x94_\xf4t\x801\x8a=ou\x83\xb3*\xe7n/\x8c:\xe2\x19yOԶ\xd2*\x03\x13͋hK\xa2m\xeaQp\x9f\x0eͺ.w\xd1\xdf:Ik\xa7,J\x872ϑG\xd2\x1d\x11\x19}\xf8\xdaY\xa2F\xed\x82\x7f\xa7H\xeb182:\xafQ\x96|\x98\x0e\x9c\x8c\xee\x95k\x1d\xe6\x98\a\xe6\xc2-\xfdP\x93Ι\xe3u4\xa2\xfc\xcf\xe6ڔB^SG\xec\xed\v\xbaC^\x8b\xc7ң\xc6\xca\xf1N\xfaU\xe8\xac\xe5^\xf3\xc1\xe7\xd4)\xda\xf8\xd1\xd0c\xee\xfe\x9e\by\xd7R\xd9\xce2\xceL'\xbaR\xf9\xef\f[\vml\x17\rs \xb1j\x14\xd4\x11\xa1\xa7\xfc\xa0\xf5ё\xe7g\u05fa\xb3\xa0\xb8QO>qzN\xbc\x1dH\xba\xe1[\xf0\x99\xab 3UKZ\nC=\x80\xdd̀\xe8X\xe3\xac@\xa2\xbd\xeb4\x96u\x99N\x90\x05I\x92\x90\x93\xebf\xdd&?p\x91\xb6nŎc\xab=\x94\xc39V\x8e\x9fG!\xc1\xb3\x9bN_\U000af8acK\xc6K\xe4!\xb9\x1d\xa2\x84&\xa3ޱ\xbbI\xfb\xc4\x16d\xb4\xac\xc2YV\x15`\xc1\xa7m\xce\xc0#S҈\x1c\x1a\xd3\xefE@I\xc6ٚ\x8b\xa2\xd63\xb4\xeal\x92\xcf\r¼69}d\x95\x8eȂH\x94\xb8\xce>\xc3\v\x9e\xd6\xf8\x95\x9e\xe7Ǧ8\x8c\x1a\xe6\xfb\x8b\x95\x16\xca\x1d\x068\xbd\xcb\xe8ӎ\xb9\xdc}\xf7\x19\xbf\xfb\x8c\xdf}\xc69\x1d}\xf7\x19'\xcaw\x9f\xf1\xbb\xcfx\xb8|\xf7\x19S\xcaw\x9fq&\"\xdf\xcagL\xc1pAk\x9c\a*$a\x95\x98\n1\x85\xf6D_>\xe9ǟ\xd58I.\xf3\xf58ȑC<\x91\xe3\x171\xaf\xa35^Mr3\xce\xc00w\xdc)\xca\x04\x87\xf9\x04\xa7g\x02\x02\xa7?=s}\x10\xf2\tO\xcf\xf8!\xa4E\x18G\x9d\x9d\tD\x9a\x7fz\xe2\xc2'\x11\x95\xc0\xc3V\x8aK\xff\x88\x8d1&I\tx|\xe3\xe4\xf7\xbd\x8c\xc9\x17\x90\xa5W9\x913K\x9eFY\x7f\xfe\xc7\xf3_\a\x8bN˔(\x1b\xf6i\xeb\xd4xL?b,\xdfM\x8d\xecg\xa9\xfez\xa6\xc2Ie?\xf5DMC\xe4\b\xbc\xbeX\x0f\xa8\xfck\xd27\x16\xcaϕ\xb7\x96'8a\x7f=\x02/\xe9\x8c=7;\x99m\xb4\x92\xaa6~M\ba\xbd\xcbܽ\x03\x01dL\xd8G5\xc8\x7f\xb0\x8d\xaa#\xa76&H\x9b\x90E\x9bF\x90^R\xadO\x8c\x00˷o\x97\xfd_\xac\xf2)\xb6\xecI\xd8M\x04\x18\xddG\xc1\xf3\x1c\xe3\x82\u0381\x1e\xaf\a\xc2UIC\xa1\x8c\x00S\x9aIQ8\x89\r\x10z\xf2\xca>Wnu\xf0h\xbfiz\r+=\x11wn\xfam\x93-9\xed\xbe?#\xe9\xf6\xa4G\xa3\xbeYZ\xedqɴ\xa9+\x94\t\x89\xb3\xe9\xe9\xb2)lu%=I69BNM\x88\x9d\xbb\x02\xf1\xa2ɯ/\x93\xf2\x9aL\xb3\xb4\xf4ֹ\x14{\x95T\xd6WN`}\xbd\xb4\xd5\x19ɪ\xa7?\xf5\x92\xbe\x96~tveڲ\xcc\xe1\x84Ӥ4Ӥ\xa5\x9b\x94\x01\x1f5Ԥ\xf4ѹI\xa3I\x9cL\x9f\xae\xaf\x9a\x16\xfa\xaaɠ\xaf\x9f\x02:)m\x93\x15\xe6&y\x8e_r\x18ʴ\x03P|\v\xe1|.\x99\x94\xee\xb9\xe6ϊ;?\x0f`\xa1\xb0\x047\xf5\x15〲.\xac\xa8\x8a\xf6>\xb6X\xc0\xb9\x81]sY\xd1ϊ\x8e\xc8\xfb\x9b\xba>\x7fi$~9\x88j\xb8aOP\x14\x8c\xc7\xe6\xe6\x1e\x152w\x0fh\xa6\x16\x80\xb6\x11g\xb9\xbf\x8c\xc9_\x1ez\xe1\xa6\v\xdd\x06@\x16\xb6\x8c-\xf5qy\xf8\xa6\xaf\x83\x06,U\x8f\xedy\xe6.ޠo\xbfԠw\x8c\xee\x1dk|\xb3\xf6P\xa9\x9f\xe8\x06\x03Ӡ~\xbc:<\xb4g\xb2\x17\xe0\xb4ꁽ\x93\xce#\x18\xe2DmP\xef\xb4\x01\x1d*U\x8cӢ\xfdD@H\xd5@\x884Mq\xfe眲|\x89\xf0\xee\x14\x01^\x92\a4\xcf{\xfd\x86\xa7'\x8f=5\x99\x9e\x8c\x92tJ\xf2%½9\x01\xdf,\x7f5\xfd\x14\xe4\xfc\x8d\xe7\x17>\xf5\xf8R\xa7\x1dgP/\xf5t\xe3|ڽ\xd2i\xc6W?\xc5\xf8\x9a\xa7\x17g\x9dZLNϚ\x95q0'\xb5\xea\x19\xc7\xed\xd2r\t\xa6O!&\x9e>L\xcc4H\x1b\xfc\x91\xc3N<]8\xffTa\"\x7f\xe7L\xe9W>=\xf8ʧ\x06\xbf\xc5i\xc1\x04\tL\xa82\xffT\u0cf7\xa4\x94\xceAOn\xfb͑\xdaIyM\x8d\xe5\xfa\x88\r\xf6\xb5\xc2m\xb2X\xab\x17\x03\x90Y\xf2\x17\xf9ӣ\r\x87\xb6\xc1Q2;\x1eQo_\xb2u\xd7\xfa\x0e\xb1\x7f\xcd\xc1m]\x1a\xa88\x1a\x00\n\xdc(5+\xea*|\xe0\xd9f\xd0Æ\x1b\xb6V\xba䖝7\x9b\xc5o\\\a\xf8\xf7\xf9\x92\xb1\x1fT\x93\xabӽ/͈\xb2*v\x18\x89\xb1\xf3n\x83\xe7IIT:C\xcf7\xaa\x10Y\xc4\xe7\x1c\xbdW\xcf5ػl\x88n\xfe\xcb:\xd9\"\xb1\xc0\a\x9b\x8bp\xebb\xffJfw\x9f\xfb\x91k%\xbc\x12\x7f\xa6'\x95N\xb0\xea\xf6\xee\xe6\x9a`\x051\xa2\xb7\x9a\x9a\x04ņ\xe5+@\x97\xa1\x1d\xfb!}r\xbd\xeeA\xed\xe7\bw\x1f\xab\x80ܽL\x12\xdc\x16\xaf\x9a3\x85Z\xeb\xe6\xda\xe1r\xa8'\x94/.wL\xf9\xa7'\x84\xce\x17\x15\xd7v璉.zx\x04\xbb>\xb5jv\xd0Z\xed\xbf\xbc\xd2-=\xb2\x87GWh'{W\xf5\x93\a\x86\xf4|\x0eN\x87OUO\x9e\xa7~\x01\x9c\x0e\xbbP\v\xa2b\xe4\xa7h\x06\xe4\xc9W,\x8d\xbf\xa1\xffG\xb5\x85\xf7ѕ\xcb\xfe\xeb+\x83&#\xa9\x89\x01*]2\x1f\xa1`\x9b\x8fHw|?O\xed\xc5s\r\x03*\xfe\x8e\xf0\xe7,N\xde\xf6A\x8d?HB7\xa8\x87Nc^\x15=\xf5\xb4c7\xf7\x14\xb76\xaa\xd4O}\x1f\xb7\x86\xe5ɐ`\x10\x81%\xe4\xc17ZNEF\xab4\x7f\x80\x8fʽ\xad\x93\"&\xfd\x16\xbd\x97\x97\xbc\xe7\x16\xf2\xb5\xfd$\x8c)z?\xb6!\xc0\xf6|\xc6\xdeE\xff\x88\xed\x91O\x19X[\xba\x91ғ&\xef\xfd\xeb$\xa8\x8f\r \v\x02\x05\x1c\xb4\x15\xfew\xa3\x9e\xe8\x02\xfc\xf8\x1asx@\xa4\xf3\x86\x19\xd0A\x11J\xe1=j\x98uU(\x9e\x83\xbe\xa2GT\x12F\xfcS\xaf\xc1\xc0\x1d\xe8?\xc5\xe2\xedfd<\xa1\xe7\x17̒A\x8f\xae(\xa0\xf8A\x14`\x1c≦\xe1f\xbfec)\xear\xe5<\xd55\xfe\xd8tr\xc02\xbb\xa1\xd2\x06C\x05\x1a\xfdD\xb7\x15Q\x9b \xf9\x87\x89\xc1\x1a>\ni\xe1\x01\xc6c\xe8\t\x9b\xe0\xdeh \a (0\x8a\xf8\xfe\x12[y\xec\x11\xe4>\xdez \x03\xcdbdL\x8e\x95w\xabn\xee\xaf\f\xabeN\x1b\x00\xf7\x7f\xbe=J~\xb7\xbd\xf7e\x82NHQ\xef\xf7\xe3-;!BG;\x91O\x1fW\xe21X\xdc\x18\x95\t\x8a*\x9e\x84\xf5\xd79\xbe\xdc\x1d\xe2\x87\x02\xc4\x03\xd2Q\x1b\xf8\xfc$A\x7f\t\x16\xc8\\\xcbػ-\xd3\xda\xef\xa7=h\xd1\xf7Z\xac¾G`\f\x000\x15\xf6\xb9\x8c{\t(l\xaf\t\xd3H\x1c\xc4>\x01\xdcyG\xc8ik\x85\xb4\xe3\x9c!n\x9bVt\xd8tDCN\x8b\xed\xfd\x00\xc6 \x93\x9d\x1e}j\xaa\xb8Ӧ\x86\xfd^\x8cy\xa3\xb4c\x96\xe1@\xff\xb0\xf7kT\x83\x1f\xd4\xde1\xcd=\xaaF\xf6>\xd2CxyGr\xbc\x97\xde\xfdR\xaf\xda\a\x15\xd8\xdf\xfe~\xf6\x8f\x00\x00\x00\xff\xff)\x00\x87w>{\x00\x00"), []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcV\xcfo+5\x10\xbe\xe7\xaf\x18\x89+\xbb\xa1B \x94\x1b*\x1c*\xe0\xa9j\x9ezw\xbc\x93d\xa8\xd7^f\xc6)A\xfc\xf1\xc8\xf6n\x9b\xee:\xb4\x8f\x03\xbe\xad\xed\xf9\xe6\x9bo~x\x9b\xa6Y\x99\x81\x1e\x91\x85\x82߀\x19\b\xffT\xf4\xe9Kڧ\x1f\xa4\xa5\xb0>ݬ\x9e\xc8w\x1b\xb8\x8d\xa2\xa1\x7f@\t\x91-\xfe\x84{\xf2\xa4\x14\xfc\xaaG5\x9dQ\xb3Y\x01\x18\uf0da\xb4-\xe9\x13\xc0\x06\xaf\x1c\x9cCn\x0e\xe8ۧ\xb8\xc3]$\xd7!g\xf0\xc9\xf5\xe9\x9b\xf6\xe6\xfb\xf6\xbb\x15\x807=n@\x90ә\x1a\x8d\xc2\xf8GDQiO\xe8\x90CKa%\x03ڄ\x7f\xe0\x10\x87\r\xbc\x1e\x14\xfb\xd1w\xe1\xbd\xcdP\xdb\f\xf5P\xa0\xf2\xa9#\xd1_\xae\xdd\xf8\x95\xc6[\x83\x8bl\\\x9dP\xbe \xc7\xc0\xfa\xe9\xd5i\x03\"\\N\xc8\x1f\xa23\\5^\x01\x88\r\x03n \xdb\x0e\xc6b\xb7\x02\x18\x05\xc9Xͨ\xc5\xe9\xa6\xc0\xd9#\xf6\xa68\x01\b\x03\xfa\x1f\xef\xef\x1e\xbfݾ\xd9\x06\xe8P,ӠYֿ\x9b\x97}\xa8\x85\t$``\xa4\x04\x1a\xc0X\x8b\"`#3z\x85B\x19\xc8\xef\x03\xf79\xad`v!\xea\x05\xaa\x1e\x11\x1e\xb3\xfec\x98\xed\xcb\xe1\xc0a@V\x9a\xa4)\xeb\xa2\xe2.v\xff\x8dxZ)\xd6b\x05]*=\x94\xecy\xd4\v\xbbQ\x1e\b{\xd0#\t0\x0e\x8c\x82\xbe\x14c\xda6\x1e\xc2\xeew\xb4\xdaΠ\x8b.\x922\x19]\x97*\xf6\x84\xac\xc0h\xc3\xc1\xd3_/ؒ\x04JN\x9dѬ\x9dWdo\x1c\x9c\x8c\x8b\xf85\x18\xdf͐{s\x06\xc6\xe4\x13\xa2\xbf\xc0\xcb\x062\xe7\xf1[`\xccRo\xe0\xa8:\xc8f\xbd>\x90N}hC\xdfGOz^疢]\xd4\xc0\xb2\xee\xf0\x84n-th\f\xdb#)Z\x8d\x8ck3P\x93\x03\xf1\xb9\x17۾\xfb\x8a\xc7Ε7n\xf5\x9cjP\x94\xc9\x1f.\x0er\xeb|AzR#\x95b*P%\xc4\xd7,\xa4\xad$\xdd\xc3\xcf\xdb\xcf01)\x99*Iy\xbd\xba\xd0e\xcaOR\x93\xfc\x1e\xb9\xd8\xed9\xf4\x19\x13}7\x04\xf2\x9a?\xac\xa3\\\xb8qד\xcaT\xda)us\xd8\xdb<\xab`\x87\x10\x87\xce(v\xf3\vw\x1enM\x8f\xee\xd6\b\xfeϹJY\x91&%\xe1Cٺ\x9c\xc0\xf3\xcbEދ\x83iv^ImeJl\a\xb4)\xb9I\xdfdM{\xb2\xa5\xad\xf6\x81\xc1\xd4L\xda\x0f1\xc9\x16_\xc8e\x9cH\x85\xcdlN\xa5.\x7f\x9fM},哣\x11\x9co\xce8ݧ;s\xff\x8e\xf6h\xcf\xd6a\x81(S\bߧ\x92\x16\xfa\xd8/}6\xf0\t\x9f+\xbb\xf7\x1c҄\xc6\xf9\xa8\xb9Z\x1bP\x1e\xb1\x03\xf9E\xb8\xf3\xc8ʭ\xfc0.G~\x0eh\x04\x02\x8eާ\x96\x0e~\x01Yy\x11\x16wH\xb1\xaf\xb0\xa9\xf2\xb9\xf3\xfb\x90\xff\"Lrl\xb4\xb4\x13\x8e\xc9\x1e\xfd\x14^\x15\xc0\xeb\xb9.k9\xe7>$hY\xf9y\xfeo\xc6i.\x11c\xd5w\x93YU\x0f\x92ǚ\xe2\xf5\xfe\x1aYF\xe7\xcc\xce\xe1\x06\x94\xe3Һ\xd8\x1afs\x9eW\xcdTj\x9f\xa9GQ\xd3\x0f\xef\x14\xd0\xe2UH\xeb~\x81\x92\x9a\xe7\xf9\x88\xfeZ\x8b\xc0\xb3\x91W\xe7\x15\xc8\xdd\xf9\x9a\xe9\xed\xcb\xdf\xe6\xb2\xcfJ=o \xcd\xfaF\xa9\"䇔\xaa\xa6\xb4\xd4y\xf5\xb7f\xa1\xd2\xf6\xf2\xee4H\xde\xf4\xcb\xf4W\xb3\x8c\xe1*\x85j\x05,63|w\x11\x9eh`s\x98\x02\xfe'\x00\x00\xff\xff\xef\xf8\xa6>\x10\f\x00\x00"), diff --git a/pkg/apis/velero/v1/labels_annotations.go b/pkg/apis/velero/v1/labels_annotations.go index 78d231ba9..a50018dbe 100644 --- a/pkg/apis/velero/v1/labels_annotations.go +++ b/pkg/apis/velero/v1/labels_annotations.go @@ -101,6 +101,9 @@ const ( // ExcludeFromBackupLabel is the label to exclude k8s resource from backup, // even if the resource contains a matching selector label. ExcludeFromBackupLabel = "velero.io/exclude-from-backup" + + // PVRLabel is the label key used to identify the pvb for pvr pod + PVRLabel = "velero.io/pod-volume-restore" ) type AsyncOperationIDPrefix string diff --git a/pkg/apis/velero/v1/pod_volume_restore_type.go b/pkg/apis/velero/v1/pod_volume_restore_type.go index 34bc7e530..2d059a14a 100644 --- a/pkg/apis/velero/v1/pod_volume_restore_type.go +++ b/pkg/apis/velero/v1/pod_volume_restore_type.go @@ -54,15 +54,23 @@ type PodVolumeRestoreSpec struct { // +optional // +nullable UploaderSettings map[string]string `json:"uploaderSettings,omitempty"` + + // Cancel indicates request to cancel the ongoing PodVolumeRestore. It can be set + // when the PodVolumeRestore is in InProgress phase + Cancel bool `json:"cancel,omitempty"` } // PodVolumeRestorePhase represents the lifecycle phase of a PodVolumeRestore. -// +kubebuilder:validation:Enum=New;InProgress;Completed;Failed +// +kubebuilder:validation:Enum=New;Accepted;Prepared;InProgress;Canceling;Canceled;Completed;Failed type PodVolumeRestorePhase string const ( PodVolumeRestorePhaseNew PodVolumeRestorePhase = "New" + PodVolumeRestorePhaseAccepted PodVolumeRestorePhase = "Accepted" + PodVolumeRestorePhasePrepared PodVolumeRestorePhase = "Prepared" PodVolumeRestorePhaseInProgress PodVolumeRestorePhase = "InProgress" + PodVolumeRestorePhaseCanceling PodVolumeRestorePhase = "Canceling" + PodVolumeRestorePhaseCanceled PodVolumeRestorePhase = "Canceled" PodVolumeRestorePhaseCompleted PodVolumeRestorePhase = "Completed" PodVolumeRestorePhaseFailed PodVolumeRestorePhase = "Failed" ) @@ -95,6 +103,16 @@ type PodVolumeRestoreStatus struct { // about the restore operation. // +optional Progress shared.DataMoveOperationProgress `json:"progress,omitempty"` + + // AcceptedTimestamp records the time the pod volume restore is to be prepared. + // The server's time is used for AcceptedTimestamp + // +optional + // +nullable + AcceptedTimestamp *metav1.Time `json:"acceptedTimestamp,omitempty"` + + // Node is name of the node where the pod volume restore is processed. + // +optional + Node string `json:"node,omitempty"` } // TODO(2.0) After converting all resources to use the runtime-controller client, the genclient and k8s:deepcopy markers will no longer be needed and should be removed. @@ -103,14 +121,14 @@ type PodVolumeRestoreStatus struct { // +kubebuilder:object:generate=true // +kubebuilder:object:root=true // +kubebuilder:storageversion -// +kubebuilder:printcolumn:name="Namespace",type="string",JSONPath=".spec.pod.namespace",description="Namespace of the pod containing the volume to be restored" -// +kubebuilder:printcolumn:name="Pod",type="string",JSONPath=".spec.pod.name",description="Name of the pod containing the volume to be restored" +// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.phase",description="PodVolumeRestore status such as New/InProgress" +// +kubebuilder:printcolumn:name="Started",type="date",JSONPath=".status.startTimestamp",description="Time duration since this PodVolumeRestore was started" +// +kubebuilder:printcolumn:name="Bytes Done",type="integer",format="int64",JSONPath=".status.progress.bytesDone",description="Completed bytes" +// +kubebuilder:printcolumn:name="Total Bytes",type="integer",format="int64",JSONPath=".status.progress.totalBytes",description="Total bytes" +// +kubebuilder:printcolumn:name="Storage Location",type="string",JSONPath=".spec.backupStorageLocation",description="Name of the Backup Storage Location where the backup data is stored" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="Time duration since this PodVolumeRestore was created" +// +kubebuilder:printcolumn:name="Node",type="string",JSONPath=".status.node",description="Name of the node where the PodVolumeRestore is processed" // +kubebuilder:printcolumn:name="Uploader Type",type="string",JSONPath=".spec.uploaderType",description="The type of the uploader to handle data transfer" -// +kubebuilder:printcolumn:name="Volume",type="string",JSONPath=".spec.volume",description="Name of the volume to be restored" -// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.phase",description="Pod Volume Restore status such as New/InProgress" -// +kubebuilder:printcolumn:name="TotalBytes",type="integer",format="int64",JSONPath=".status.progress.totalBytes",description="Pod Volume Restore status such as New/InProgress" -// +kubebuilder:printcolumn:name="BytesDone",type="integer",format="int64",JSONPath=".status.progress.bytesDone",description="Pod Volume Restore status such as New/InProgress" -// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" type PodVolumeRestore struct { metav1.TypeMeta `json:",inline"` diff --git a/pkg/apis/velero/v1/zz_generated.deepcopy.go b/pkg/apis/velero/v1/zz_generated.deepcopy.go index 2fd13cfd2..47cc8b199 100644 --- a/pkg/apis/velero/v1/zz_generated.deepcopy.go +++ b/pkg/apis/velero/v1/zz_generated.deepcopy.go @@ -1149,6 +1149,10 @@ func (in *PodVolumeRestoreStatus) DeepCopyInto(out *PodVolumeRestoreStatus) { *out = (*in).DeepCopy() } out.Progress = in.Progress + if in.AcceptedTimestamp != nil { + in, out := &in.AcceptedTimestamp, &out.AcceptedTimestamp + *out = (*in).DeepCopy() + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodVolumeRestoreStatus. diff --git a/pkg/builder/pod_builder.go b/pkg/builder/pod_builder.go index 50f8f5e51..c38acf66e 100644 --- a/pkg/builder/pod_builder.go +++ b/pkg/builder/pod_builder.go @@ -88,6 +88,11 @@ func (b *PodBuilder) InitContainers(containers ...*corev1api.Container) *PodBuil return b } +func (b *PodBuilder) InitContainerState(state corev1api.ContainerState) *PodBuilder { + b.object.Status.InitContainerStatuses = append(b.object.Status.InitContainerStatuses, corev1api.ContainerStatus{State: state}) + return b +} + func (b *PodBuilder) Containers(containers ...*corev1api.Container) *PodBuilder { for _, c := range containers { b.object.Spec.Containers = append(b.object.Spec.Containers, *c) diff --git a/pkg/builder/pod_volume_restore_builder.go b/pkg/builder/pod_volume_restore_builder.go index 3d4da94d6..965dd5a5d 100644 --- a/pkg/builder/pod_volume_restore_builder.go +++ b/pkg/builder/pod_volume_restore_builder.go @@ -97,3 +97,33 @@ func (b *PodVolumeRestoreBuilder) UploaderType(uploaderType string) *PodVolumeRe b.object.Spec.UploaderType = uploaderType return b } + +// Cancel sets the DataDownload's Cancel. +func (d *PodVolumeRestoreBuilder) Cancel(cancel bool) *PodVolumeRestoreBuilder { + d.object.Spec.Cancel = cancel + return d +} + +// AcceptedTimestamp sets the PodVolumeRestore's AcceptedTimestamp. +func (d *PodVolumeRestoreBuilder) AcceptedTimestamp(acceptedTimestamp *metav1.Time) *PodVolumeRestoreBuilder { + d.object.Status.AcceptedTimestamp = acceptedTimestamp + return d +} + +// Finalizers sets the PodVolumeRestore's Finalizers. +func (d *PodVolumeRestoreBuilder) Finalizers(finalizers []string) *PodVolumeRestoreBuilder { + d.object.Finalizers = finalizers + return d +} + +// Message sets the PodVolumeRestore's Message. +func (d *PodVolumeRestoreBuilder) Message(msg string) *PodVolumeRestoreBuilder { + d.object.Status.Message = msg + return d +} + +// Message sets the PodVolumeRestore's Node. +func (d *PodVolumeRestoreBuilder) Node(node string) *PodVolumeRestoreBuilder { + d.object.Status.Node = node + return d +} diff --git a/pkg/cmd/cli/nodeagent/server.go b/pkg/cmd/cli/nodeagent/server.go index ede158116..286ca7439 100644 --- a/pkg/cmd/cli/nodeagent/server.go +++ b/pkg/cmd/cli/nodeagent/server.go @@ -306,10 +306,6 @@ func (s *nodeAgentServer) run() { s.logger.Fatal(err, "unable to create controller", "controller", constant.ControllerPodVolumeBackup) } - if err = controller.NewPodVolumeRestoreReconciler(s.mgr.GetClient(), s.kubeClient, s.dataPathMgr, repoEnsurer, credentialGetter, s.logger).SetupWithManager(s.mgr); err != nil { - s.logger.WithError(err).Fatal("Unable to create the pod volume restore controller") - } - var loadAffinity *kube.LoadAffinity if s.dataPathConfigs != nil && len(s.dataPathConfigs.LoadAffinity) > 0 { loadAffinity = s.dataPathConfigs.LoadAffinity[0] @@ -332,6 +328,10 @@ func (s *nodeAgentServer) run() { } } + if err = controller.NewPodVolumeRestoreReconciler(s.mgr.GetClient(), s.mgr, s.kubeClient, s.dataPathMgr, s.nodeName, s.config.dataMoverPrepareTimeout, s.config.resourceTimeout, podResources, s.logger).SetupWithManager(s.mgr); err != nil { + s.logger.WithError(err).Fatal("Unable to create the pod volume restore controller") + } + dataUploadReconciler := controller.NewDataUploadReconciler( s.mgr.GetClient(), s.mgr, @@ -525,7 +525,7 @@ func (s *nodeAgentServer) markInProgressPVRsFailed(client ctrlclient.Client) { continue } - if err := controller.UpdatePVRStatusToFailed(s.ctx, client, &pvrs.Items[i], + if err := controller.UpdatePVRStatusToFailed(s.ctx, client, &pvrs.Items[i], errors.New("cannot survive from node-agent restart"), fmt.Sprintf("get a podvolumerestore with status %q during the server starting, mark it as %q", velerov1api.PodVolumeRestorePhaseInProgress, velerov1api.PodVolumeRestorePhaseFailed), time.Now(), s.logger); err != nil { s.logger.WithError(errors.WithStack(err)).Errorf("failed to patch podvolumerestore %q", pvr.GetName()) diff --git a/pkg/controller/data_download_controller.go b/pkg/controller/data_download_controller.go index 53d83d98e..75c04e377 100644 --- a/pkg/controller/data_download_controller.go +++ b/pkg/controller/data_download_controller.go @@ -198,9 +198,7 @@ func (r *DataDownloadReconciler) Reconcile(ctx context.Context, req ctrl.Request if time.Since(spotted) > delay { log.Infof("Data download %s is canceled in Phase %s but not handled in rasonable time", dd.GetName(), dd.Status.Phase) - if r.tryCancelDataDownload(ctx, dd, "") { - delete(r.cancelledDataDownload, dd.Name) - } + r.tryCancelDataDownload(ctx, dd, "") return ctrl.Result{}, nil } @@ -526,6 +524,7 @@ func (r *DataDownloadReconciler) tryCancelDataDownload(ctx context.Context, dd * // success update r.metrics.RegisterDataDownloadCancel(r.nodeName) r.restoreExposer.CleanUp(ctx, getDataDownloadOwnerObject(dd)) + delete(r.cancelledDataDownload, dd.Name) log.Warn("data download is canceled") diff --git a/pkg/controller/data_upload_controller.go b/pkg/controller/data_upload_controller.go index 6ab65f172..7dc19e855 100644 --- a/pkg/controller/data_upload_controller.go +++ b/pkg/controller/data_upload_controller.go @@ -226,9 +226,7 @@ func (r *DataUploadReconciler) Reconcile(ctx context.Context, req ctrl.Request) if time.Since(spotted) > delay { log.Infof("Data upload %s is canceled in Phase %s but not handled in reasonable time", du.GetName(), du.Status.Phase) - if r.tryCancelDataUpload(ctx, du, "") { - delete(r.cancelledDataUpload, du.Name) - } + r.tryCancelDataUpload(ctx, du, "") return ctrl.Result{}, nil } @@ -566,6 +564,7 @@ func (r *DataUploadReconciler) tryCancelDataUpload(ctx context.Context, du *vele r.metrics.RegisterDataUploadCancel(r.nodeName) // cleans up any objects generated during the snapshot expose r.cleanUp(ctx, du, log) + delete(r.cancelledDataUpload, du.Name) log.Warn("data upload is canceled") diff --git a/pkg/controller/pod_volume_backup_controller.go b/pkg/controller/pod_volume_backup_controller.go index 254a39c91..16ad49c06 100644 --- a/pkg/controller/pod_volume_backup_controller.go +++ b/pkg/controller/pod_volume_backup_controller.go @@ -46,7 +46,10 @@ import ( "github.com/vmware-tanzu/velero/pkg/util/filesystem" ) -const pVBRRequestor string = "pod-volume-backup-restore" +const ( + pVBRRequestor = "pod-volume-backup-restore" + PodVolumeFinalizer = "velero.io/pod-volume-finalizer" +) // NewPodVolumeBackupReconciler creates the PodVolumeBackupReconciler instance func NewPodVolumeBackupReconciler(client client.Client, kubeClient kubernetes.Interface, dataPathMgr *datapath.Manager, ensurer *repository.Ensurer, credentialGetter *credentials.CredentialGetter, diff --git a/pkg/controller/pod_volume_restore_controller.go b/pkg/controller/pod_volume_restore_controller.go index e65d1b606..46a00937d 100644 --- a/pkg/controller/pod_volume_restore_controller.go +++ b/pkg/controller/pod_volume_restore_controller.go @@ -19,8 +19,7 @@ package controller import ( "context" "fmt" - "os" - "path/filepath" + "strings" "time" "github.com/pkg/errors" @@ -30,49 +29,64 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" clocks "k8s.io/utils/clock" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "github.com/vmware-tanzu/velero/internal/credentials" veleroapishared "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" + velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/constant" "github.com/vmware-tanzu/velero/pkg/datapath" "github.com/vmware-tanzu/velero/pkg/exposer" - "github.com/vmware-tanzu/velero/pkg/podvolume" - "github.com/vmware-tanzu/velero/pkg/repository" + "github.com/vmware-tanzu/velero/pkg/nodeagent" "github.com/vmware-tanzu/velero/pkg/restorehelper" "github.com/vmware-tanzu/velero/pkg/uploader" - "github.com/vmware-tanzu/velero/pkg/util/boolptr" - "github.com/vmware-tanzu/velero/pkg/util/filesystem" + "github.com/vmware-tanzu/velero/pkg/util" + "github.com/vmware-tanzu/velero/pkg/util/kube" ) -func NewPodVolumeRestoreReconciler(client client.Client, kubeClient kubernetes.Interface, dataPathMgr *datapath.Manager, ensurer *repository.Ensurer, - credentialGetter *credentials.CredentialGetter, logger logrus.FieldLogger) *PodVolumeRestoreReconciler { +func NewPodVolumeRestoreReconciler(client client.Client, mgr manager.Manager, kubeClient kubernetes.Interface, dataPathMgr *datapath.Manager, + nodeName string, preparingTimeout time.Duration, resourceTimeout time.Duration, podResources corev1api.ResourceRequirements, + logger logrus.FieldLogger) *PodVolumeRestoreReconciler { return &PodVolumeRestoreReconciler{ - Client: client, - kubeClient: kubeClient, - logger: logger.WithField("controller", "PodVolumeRestore"), - repositoryEnsurer: ensurer, - credentialGetter: credentialGetter, - fileSystem: filesystem.NewFileSystem(), - clock: &clocks.RealClock{}, - dataPathMgr: dataPathMgr, + client: client, + mgr: mgr, + kubeClient: kubeClient, + logger: logger.WithField("controller", "PodVolumeRestore"), + nodeName: nodeName, + clock: &clocks.RealClock{}, + podResources: podResources, + dataPathMgr: dataPathMgr, + preparingTimeout: preparingTimeout, + resourceTimeout: resourceTimeout, + exposer: exposer.NewPodVolumeExposer(kubeClient, logger), + cancelledPVR: make(map[string]time.Time), } } type PodVolumeRestoreReconciler struct { - client.Client - kubeClient kubernetes.Interface - logger logrus.FieldLogger - repositoryEnsurer *repository.Ensurer - credentialGetter *credentials.CredentialGetter - fileSystem filesystem.Interface - clock clocks.WithTickerAndDelayedExecution - dataPathMgr *datapath.Manager + client client.Client + mgr manager.Manager + kubeClient kubernetes.Interface + logger logrus.FieldLogger + nodeName string + clock clocks.WithTickerAndDelayedExecution + podResources corev1api.ResourceRequirements + exposer exposer.PodVolumeExposer + dataPathMgr *datapath.Manager + preparingTimeout time.Duration + resourceTimeout time.Duration + cancelledPVR map[string]time.Time } // +kubebuilder:rbac:groups=velero.io,resources=podvolumerestores,verbs=get;list;watch;create;update;patch;delete @@ -81,123 +95,428 @@ type PodVolumeRestoreReconciler struct { // +kubebuilder:rbac:groups="",resources=persistentvolumes,verbs=get // +kubebuilder:rbac:groups="",resources=persistentvolumerclaims,verbs=get -func (c *PodVolumeRestoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - log := c.logger.WithField("PodVolumeRestore", req.NamespacedName.String()) +func (r *PodVolumeRestoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := r.logger.WithField("PodVolumeRestore", req.NamespacedName.String()) + log.Info("Reconciling PVR by advanced controller") pvr := &velerov1api.PodVolumeRestore{} - if err := c.Get(ctx, types.NamespacedName{Namespace: req.Namespace, Name: req.Name}, pvr); err != nil { + if err := r.client.Get(ctx, types.NamespacedName{Namespace: req.Namespace, Name: req.Name}, pvr); err != nil { if apierrors.IsNotFound(err) { - log.Warn("PodVolumeRestore not found, skip") + log.Warn("PVR not found, skip") return ctrl.Result{}, nil } - log.WithError(err).Error("Unable to get the PodVolumeRestore") + log.WithError(err).Error("Unable to get the PVR") return ctrl.Result{}, err } + log = log.WithField("pod", fmt.Sprintf("%s/%s", pvr.Spec.Pod.Namespace, pvr.Spec.Pod.Name)) if len(pvr.OwnerReferences) == 1 { log = log.WithField("restore", fmt.Sprintf("%s/%s", pvr.Namespace, pvr.OwnerReferences[0].Name)) } - shouldProcess, pod, err := c.shouldProcess(ctx, log, pvr) - if err != nil { - return ctrl.Result{}, err - } - if !shouldProcess { - return ctrl.Result{}, nil - } + // Logic for clear resources when pvr been deleted + if !isPVRInFinalState(pvr) { + if !controllerutil.ContainsFinalizer(pvr, PodVolumeFinalizer) { + if err := UpdatePVRWithRetry(ctx, r.client, req.NamespacedName, log, func(pvr *velerov1api.PodVolumeRestore) bool { + if controllerutil.ContainsFinalizer(pvr, PodVolumeFinalizer) { + return false + } - initContainerIndex := getInitContainerIndex(pod) - if initContainerIndex > 0 { - log.Warnf(`Init containers before the %s container may cause issues - if they interfere with volumes being restored: %s index %d`, restorehelper.WaitInitContainer, restorehelper.WaitInitContainer, initContainerIndex) - } + controllerutil.AddFinalizer(pvr, PodVolumeFinalizer) - log.Info("Restore starting") + return true + }); err != nil { + log.WithError(err).Errorf("failed to add finalizer for PVR %s/%s", pvr.Namespace, pvr.Name) + return ctrl.Result{}, err + } - callbacks := datapath.Callbacks{ - OnCompleted: c.OnDataPathCompleted, - OnFailed: c.OnDataPathFailed, - OnCancelled: c.OnDataPathCancelled, - OnProgress: c.OnDataPathProgress, - } + return ctrl.Result{}, nil + } - fsRestore, err := c.dataPathMgr.CreateFileSystemBR(pvr.Name, pVBRRequestor, ctx, c.Client, pvr.Namespace, callbacks, log) - if err != nil { - if err == datapath.ConcurrentLimitExceed { - return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil - } else { - return c.errorOut(ctx, pvr, err, "error to create data path", log) + if !pvr.DeletionTimestamp.IsZero() { + if !pvr.Spec.Cancel { + log.Warnf("Cancel PVR under phase %s because it is being deleted", pvr.Status.Phase) + + if err := UpdatePVRWithRetry(ctx, r.client, req.NamespacedName, log, func(pvr *velerov1api.PodVolumeRestore) bool { + if pvr.Spec.Cancel { + return false + } + + pvr.Spec.Cancel = true + pvr.Status.Message = "Cancel PVR because it is being deleted" + + return true + }); err != nil { + log.WithError(err).Errorf("failed to set cancel flag for PVR %s/%s", pvr.Namespace, pvr.Name) + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil + } + } + } else { + delete(r.cancelledPVR, pvr.Name) + + if controllerutil.ContainsFinalizer(pvr, PodVolumeFinalizer) { + if err := UpdatePVRWithRetry(ctx, r.client, req.NamespacedName, log, func(pvr *velerov1api.PodVolumeRestore) bool { + if !controllerutil.ContainsFinalizer(pvr, PodVolumeFinalizer) { + return false + } + + controllerutil.RemoveFinalizer(pvr, PodVolumeFinalizer) + + return true + }); err != nil { + log.WithError(err).Error("error to remove finalizer") + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil } } - original := pvr.DeepCopy() - pvr.Status.Phase = velerov1api.PodVolumeRestorePhaseInProgress - pvr.Status.StartTimestamp = &metav1.Time{Time: c.clock.Now()} - if err = c.Patch(ctx, pvr, client.MergeFrom(original)); err != nil { - c.closeDataPath(ctx, pvr.Name) - return c.errorOut(ctx, pvr, err, "error to update status to in progress", log) + if pvr.Spec.Cancel { + if spotted, found := r.cancelledPVR[pvr.Name]; !found { + r.cancelledPVR[pvr.Name] = r.clock.Now() + } else { + delay := cancelDelayOthers + if pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseInProgress { + delay = cancelDelayInProgress + } + + if time.Since(spotted) > delay { + log.Infof("PVR %s is canceled in Phase %s but not handled in rasonable time", pvr.GetName(), pvr.Status.Phase) + r.tryCancelPodVolumeRestore(ctx, pvr, "") + + return ctrl.Result{}, nil + } + } } - volumePath, err := exposer.GetPodVolumeHostPath(ctx, pod, pvr.Spec.Volume, c.kubeClient, c.fileSystem, log) - if err != nil { - c.closeDataPath(ctx, pvr.Name) - return c.errorOut(ctx, pvr, err, "error exposing host path for pod volume", log) + if pvr.Status.Phase == "" || pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseNew { + if pvr.Spec.Cancel { + log.Infof("PVR %s is canceled in Phase %s", pvr.GetName(), pvr.Status.Phase) + r.tryCancelPodVolumeRestore(ctx, pvr, "") + + return ctrl.Result{}, nil + } + + shouldProcess, pod, err := shouldProcess(ctx, r.client, log, pvr) + if err != nil { + return ctrl.Result{}, err + } + if !shouldProcess { + return ctrl.Result{}, nil + } + + log.Info("Accepting PVR") + + if err := r.acceptPodVolumeRestore(ctx, pvr); err != nil { + return ctrl.Result{}, errors.Wrapf(err, "error accepting PVR %s", pvr.Name) + } + + initContainerIndex := getInitContainerIndex(pod) + if initContainerIndex > 0 { + log.Warnf(`Init containers before the %s container may cause issues + if they interfere with volumes being restored: %s index %d`, restorehelper.WaitInitContainer, restorehelper.WaitInitContainer, initContainerIndex) + } + + log.Info("Exposing PVR") + + exposeParam := r.setupExposeParam(pvr) + if err := r.exposer.Expose(ctx, getPVROwnerObject(pvr), exposeParam); err != nil { + return r.errorOut(ctx, pvr, err, "error to expose PVR", log) + } + + log.Info("PVR is exposed") + + return ctrl.Result{}, nil + } else if pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseAccepted { + if peekErr := r.exposer.PeekExposed(ctx, getPVROwnerObject(pvr)); peekErr != nil { + log.Errorf("Cancel PVR %s/%s because of expose error %s", pvr.Namespace, pvr.Name, peekErr) + r.tryCancelPodVolumeRestore(ctx, pvr, fmt.Sprintf("found a PVR %s/%s with expose error: %s. mark it as cancel", pvr.Namespace, pvr.Name, peekErr)) + } else if pvr.Status.AcceptedTimestamp != nil { + if time.Since(pvr.Status.AcceptedTimestamp.Time) >= r.preparingTimeout { + r.onPrepareTimeout(ctx, pvr) + } + } + + return ctrl.Result{}, nil + } else if pvr.Status.Phase == velerov1api.PodVolumeRestorePhasePrepared { + log.Infof("PVR is prepared and should be processed by %s (%s)", pvr.Status.Node, r.nodeName) + + if pvr.Status.Node != r.nodeName { + return ctrl.Result{}, nil + } + + if pvr.Spec.Cancel { + log.Info("Prepared PVR is being cancelled") + r.OnDataPathCancelled(ctx, pvr.GetNamespace(), pvr.GetName()) + return ctrl.Result{}, nil + } + + asyncBR := r.dataPathMgr.GetAsyncBR(pvr.Name) + if asyncBR != nil { + log.Info("Cancellable data path is already started") + return ctrl.Result{}, nil + } + + res, err := r.exposer.GetExposed(ctx, getPVROwnerObject(pvr), r.client, r.nodeName, r.resourceTimeout) + if err != nil { + return r.errorOut(ctx, pvr, err, "exposed PVR is not ready", log) + } else if res == nil { + return r.errorOut(ctx, pvr, errors.New("no expose result is available for the current node"), "exposed PVR is not ready", log) + } + + log.Info("Exposed PVR is ready and creating data path routine") + + callbacks := datapath.Callbacks{ + OnCompleted: r.OnDataPathCompleted, + OnFailed: r.OnDataPathFailed, + OnCancelled: r.OnDataPathCancelled, + OnProgress: r.OnDataPathProgress, + } + + asyncBR, err = r.dataPathMgr.CreateMicroServiceBRWatcher(ctx, r.client, r.kubeClient, r.mgr, datapath.TaskTypeRestore, + pvr.Name, pvr.Namespace, res.ByPod.HostingPod.Name, res.ByPod.HostingContainer, pvr.Name, callbacks, false, log) + if err != nil { + if err == datapath.ConcurrentLimitExceed { + log.Info("Data path instance is concurrent limited requeue later") + return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil + } else { + return r.errorOut(ctx, pvr, err, "error to create data path", log) + } + } + + if err := r.initCancelableDataPath(ctx, asyncBR, res, log); err != nil { + log.WithError(err).Errorf("Failed to init cancelable data path for %s", pvr.Name) + + r.closeDataPath(ctx, pvr.Name) + return r.errorOut(ctx, pvr, err, "error initializing data path", log) + } + + terminated := false + if err := UpdatePVRWithRetry(ctx, r.client, types.NamespacedName{Namespace: pvr.Namespace, Name: pvr.Name}, log, func(pvr *velerov1api.PodVolumeRestore) bool { + if isPVRInFinalState(pvr) { + terminated = true + return false + } + + pvr.Status.Phase = velerov1api.PodVolumeRestorePhaseInProgress + pvr.Status.StartTimestamp = &metav1.Time{Time: r.clock.Now()} + + return true + }); err != nil { + log.WithError(err).Warnf("Failed to update PVR %s to InProgress, will data path close and retry", pvr.Name) + + r.closeDataPath(ctx, pvr.Name) + return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, nil + } + + if terminated { + log.Warnf("PVR %s is terminated during transition from prepared", pvr.Name) + r.closeDataPath(ctx, pvr.Name) + return ctrl.Result{}, nil + } + + log.Info("PVR is marked as in progress") + + if err := r.startCancelableDataPath(asyncBR, pvr, res, log); err != nil { + log.WithError(err).Errorf("Failed to start cancelable data path for %s", pvr.Name) + r.closeDataPath(ctx, pvr.Name) + + return r.errorOut(ctx, pvr, err, "error starting data path", log) + } + + return ctrl.Result{}, nil + } else if pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseInProgress { + if pvr.Spec.Cancel { + if pvr.Status.Node != r.nodeName { + return ctrl.Result{}, nil + } + + log.Info("PVR is being canceled") + + asyncBR := r.dataPathMgr.GetAsyncBR(pvr.Name) + if asyncBR == nil { + r.OnDataPathCancelled(ctx, pvr.GetNamespace(), pvr.GetName()) + return ctrl.Result{}, nil + } + + // Update status to Canceling + if err := UpdatePVRWithRetry(ctx, r.client, types.NamespacedName{Namespace: pvr.Namespace, Name: pvr.Name}, log, func(pvr *velerov1api.PodVolumeRestore) bool { + if isPVRInFinalState(pvr) { + log.Warnf("PVR %s is terminated, abort setting it to cancelling", pvr.Name) + return false + } + + pvr.Status.Phase = velerov1api.PodVolumeRestorePhaseCanceling + return true + }); err != nil { + log.WithError(err).Error("error updating PVR into canceling status") + return ctrl.Result{}, err + } + + asyncBR.Cancel() + return ctrl.Result{}, nil + } + return ctrl.Result{}, nil } - log.WithField("path", volumePath.ByPath).Debugf("Found host path") - - if err := fsRestore.Init(ctx, &datapath.FSBRInitParam{ - BSLName: pvr.Spec.BackupStorageLocation, - SourceNamespace: pvr.Spec.SourceNamespace, - UploaderType: pvr.Spec.UploaderType, - RepositoryType: podvolume.GetPvrRepositoryType(pvr), - RepoIdentifier: pvr.Spec.RepoIdentifier, - RepositoryEnsurer: c.repositoryEnsurer, - CredentialGetter: c.credentialGetter, - }); err != nil { - c.closeDataPath(ctx, pvr.Name) - return c.errorOut(ctx, pvr, err, "error to initialize data path", log) - } - - if err := fsRestore.StartRestore(pvr.Spec.SnapshotID, volumePath, pvr.Spec.UploaderSettings); err != nil { - c.closeDataPath(ctx, pvr.Name) - return c.errorOut(ctx, pvr, err, "error starting data path restore", log) - } - - log.WithField("path", volumePath.ByPath).Info("Async fs restore data path started") - return ctrl.Result{}, nil } -func (c *PodVolumeRestoreReconciler) errorOut(ctx context.Context, pvr *velerov1api.PodVolumeRestore, err error, msg string, log logrus.FieldLogger) (ctrl.Result, error) { - _ = UpdatePVRStatusToFailed(ctx, c.Client, pvr, errors.WithMessage(err, msg).Error(), c.clock.Now(), log) - return ctrl.Result{}, err +func (r *PodVolumeRestoreReconciler) acceptPodVolumeRestore(ctx context.Context, pvr *velerov1api.PodVolumeRestore) error { + return UpdatePVRWithRetry(ctx, r.client, types.NamespacedName{Namespace: pvr.Namespace, Name: pvr.Name}, r.logger, func(pvr *velerov1api.PodVolumeRestore) bool { + pvr.Status.AcceptedTimestamp = &metav1.Time{Time: r.clock.Now()} + pvr.Status.Phase = velerov1api.PodVolumeRestorePhaseAccepted + pvr.Status.Node = r.nodeName + + return true + }) } -func UpdatePVRStatusToFailed(ctx context.Context, c client.Client, pvb *velerov1api.PodVolumeRestore, errString string, time time.Time, log logrus.FieldLogger) error { - original := pvb.DeepCopy() - pvb.Status.Phase = velerov1api.PodVolumeRestorePhaseFailed - pvb.Status.Message = errString - pvb.Status.CompletionTimestamp = &metav1.Time{Time: time} +func (r *PodVolumeRestoreReconciler) tryCancelPodVolumeRestore(ctx context.Context, pvr *velerov1api.PodVolumeRestore, message string) bool { + log := r.logger.WithField("PVR", pvr.Name) + succeeded, err := funcExclusiveUpdatePodVolumeRestore(ctx, r.client, pvr, func(pvr *velerov1api.PodVolumeRestore) { + pvr.Status.Phase = velerov1api.PodVolumeRestorePhaseCanceled + if pvr.Status.StartTimestamp.IsZero() { + pvr.Status.StartTimestamp = &metav1.Time{Time: r.clock.Now()} + } + pvr.Status.CompletionTimestamp = &metav1.Time{Time: r.clock.Now()} + + if message != "" { + pvr.Status.Message = message + } + }) - err := c.Patch(ctx, pvb, client.MergeFrom(original)) if err != nil { - log.WithError(err).Error("error updating PodVolumeRestore status") + log.WithError(err).Error("error updating PVR status") + return false + } else if !succeeded { + log.Warn("conflict in updating PVR status and will try it again later") + return false + } + + r.exposer.CleanUp(ctx, getPVROwnerObject(pvr)) + delete(r.cancelledPVR, pvr.Name) + + log.Warn("PVR is canceled") + + return true +} + +var funcExclusiveUpdatePodVolumeRestore = exclusiveUpdatePodVolumeRestore + +func exclusiveUpdatePodVolumeRestore(ctx context.Context, cli client.Client, pvr *velerov1api.PodVolumeRestore, + updateFunc func(*velerov1api.PodVolumeRestore)) (bool, error) { + updateFunc(pvr) + + err := cli.Update(ctx, pvr) + if err == nil { + return true, nil + } + + // warn we won't rollback pvr values in memory when error + if apierrors.IsConflict(err) { + return false, nil + } else { + return false, err + } +} + +func (r *PodVolumeRestoreReconciler) onPrepareTimeout(ctx context.Context, pvr *velerov1api.PodVolumeRestore) { + log := r.logger.WithField("PVR", pvr.Name) + + log.Info("Timeout happened for preparing PVR") + + succeeded, err := funcExclusiveUpdatePodVolumeRestore(ctx, r.client, pvr, func(pvr *velerov1api.PodVolumeRestore) { + pvr.Status.Phase = velerov1api.PodVolumeRestorePhaseFailed + pvr.Status.Message = "timeout on preparing PVR" + }) + + if err != nil { + log.WithError(err).Warn("Failed to update PVR") + return + } + + if !succeeded { + log.Warn("PVR has been updated by others") + return + } + + diags := strings.Split(r.exposer.DiagnoseExpose(ctx, getPVROwnerObject(pvr)), "\n") + for _, diag := range diags { + log.Warnf("[Diagnose PVR expose]%s", diag) + } + + r.exposer.CleanUp(ctx, getPVROwnerObject(pvr)) + + log.Info("PVR has been cleaned up") +} + +func (r *PodVolumeRestoreReconciler) initCancelableDataPath(ctx context.Context, asyncBR datapath.AsyncBR, res *exposer.ExposeResult, log logrus.FieldLogger) error { + log.Info("Init cancelable PVR") + + if err := asyncBR.Init(ctx, nil); err != nil { + return errors.Wrap(err, "error initializing asyncBR") + } + + log.Infof("async data path init for pod %s, volume %s", res.ByPod.HostingPod.Name, res.ByPod.VolumeName) + + return nil +} + +func (r *PodVolumeRestoreReconciler) startCancelableDataPath(asyncBR datapath.AsyncBR, pvr *velerov1api.PodVolumeRestore, res *exposer.ExposeResult, log logrus.FieldLogger) error { + log.Info("Start cancelable PVR") + + if err := asyncBR.StartRestore(pvr.Spec.SnapshotID, datapath.AccessPoint{ + ByPath: res.ByPod.VolumeName, + }, pvr.Spec.UploaderSettings); err != nil { + return errors.Wrapf(err, "error starting async restore for pod %s, volume %s", res.ByPod.HostingPod.Name, res.ByPod.VolumeName) + } + + log.Infof("Async restore started for pod %s, volume %s", res.ByPod.HostingPod.Name, res.ByPod.VolumeName) + return nil +} + +func (r *PodVolumeRestoreReconciler) errorOut(ctx context.Context, pvr *velerov1api.PodVolumeRestore, err error, msg string, log logrus.FieldLogger) (ctrl.Result, error) { + r.exposer.CleanUp(ctx, getPVROwnerObject(pvr)) + + return ctrl.Result{}, UpdatePVRStatusToFailed(ctx, r.client, pvr, err, msg, r.clock.Now(), log) +} + +func UpdatePVRStatusToFailed(ctx context.Context, c client.Client, pvr *velerov1api.PodVolumeRestore, err error, msg string, time time.Time, log logrus.FieldLogger) error { + log.Info("update PVR status to Failed") + + if patchErr := UpdatePVRWithRetry(context.Background(), c, types.NamespacedName{Namespace: pvr.Namespace, Name: pvr.Name}, log, + func(pvr *velerov1api.PodVolumeRestore) bool { + if isPVRInFinalState(pvr) { + return false + } + + pvr.Status.Phase = velerov1api.PodVolumeRestorePhaseFailed + pvr.Status.Message = errors.WithMessage(err, msg).Error() + pvr.Status.CompletionTimestamp = &metav1.Time{Time: time} + + return true + }); patchErr != nil { + log.WithError(patchErr).Warn("error updating PVR status") } return err } -func (c *PodVolumeRestoreReconciler) shouldProcess(ctx context.Context, log logrus.FieldLogger, pvr *velerov1api.PodVolumeRestore) (bool, *corev1api.Pod, error) { +func shouldProcess(ctx context.Context, client client.Client, log logrus.FieldLogger, pvr *velerov1api.PodVolumeRestore) (bool, *corev1api.Pod, error) { if !isPVRNew(pvr) { - log.Debug("PodVolumeRestore is not new, skip") + log.Debug("PVR is not new, skip") return false, nil, nil } // we filter the pods during the initialization of cache, if we can get a pod here, the pod must be in the same node with the controller // so we don't need to compare the node anymore pod := &corev1api.Pod{} - if err := c.Get(ctx, types.NamespacedName{Namespace: pvr.Spec.Pod.Namespace, Name: pvr.Spec.Pod.Name}, pod); err != nil { + if err := client.Get(ctx, types.NamespacedName{Namespace: pvr.Spec.Pod.Namespace, Name: pvr.Spec.Pod.Name}, pod); err != nil { if apierrors.IsNotFound(err) { log.WithError(err).Debug("Pod not found on this node, skip") return false, nil, nil @@ -214,39 +533,175 @@ func (c *PodVolumeRestoreReconciler) shouldProcess(ctx context.Context, log logr return true, pod, nil } -func (c *PodVolumeRestoreReconciler) SetupWithManager(mgr ctrl.Manager) error { - // The pod may not being scheduled at the point when its PVRs are initially reconciled. - // By watching the pods, we can trigger the PVR reconciliation again once the pod is finally scheduled on the node. - return ctrl.NewControllerManagedBy(mgr). - For(&velerov1api.PodVolumeRestore{}). - Watches(&corev1api.Pod{}, handler.EnqueueRequestsFromMapFunc(c.findVolumeRestoresForPod)). - Complete(c) +func (r *PodVolumeRestoreReconciler) closeDataPath(ctx context.Context, pvrName string) { + asyncBR := r.dataPathMgr.GetAsyncBR(pvrName) + if asyncBR != nil { + asyncBR.Close(ctx) + } + + r.dataPathMgr.RemoveAsyncBR(pvrName) } -func (c *PodVolumeRestoreReconciler) findVolumeRestoresForPod(ctx context.Context, pod client.Object) []reconcile.Request { +func (r *PodVolumeRestoreReconciler) SetupWithManager(mgr ctrl.Manager) error { + gp := kube.NewGenericEventPredicate(func(object client.Object) bool { + pvr := object.(*velerov1api.PodVolumeRestore) + if isLegacyPVR(pvr) { + return false + } + + if pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseAccepted { + return true + } + + if pvr.Spec.Cancel && !isPVRInFinalState(pvr) { + return true + } + + if isPVRInFinalState(pvr) && !pvr.DeletionTimestamp.IsZero() { + return true + } + + return false + }) + + s := kube.NewPeriodicalEnqueueSource(r.logger.WithField("controller", constant.ControllerPodVolumeRestore), r.client, &velerov1api.PodVolumeRestoreList{}, preparingMonitorFrequency, kube.PeriodicalEnqueueSourceOption{ + Predicates: []predicate.Predicate{gp}, + }) + + pred := kube.NewAllEventPredicate(func(obj client.Object) bool { + pvr := obj.(*velerov1.PodVolumeRestore) + return !isLegacyPVR(pvr) + }) + + return ctrl.NewControllerManagedBy(mgr). + For(&velerov1api.PodVolumeRestore{}, builder.WithPredicates(pred)). + WatchesRawSource(s). + Watches(&corev1api.Pod{}, handler.EnqueueRequestsFromMapFunc(r.findPVRForTargetPod)). + Watches(&corev1api.Pod{}, kube.EnqueueRequestsFromMapUpdateFunc(r.findPVRForRestorePod), + builder.WithPredicates(predicate.Funcs{ + UpdateFunc: func(ue event.UpdateEvent) bool { + newObj := ue.ObjectNew.(*corev1api.Pod) + + if _, ok := newObj.Labels[velerov1api.PVRLabel]; !ok { + return false + } + + if newObj.Spec.NodeName == "" { + return false + } + + return true + }, + CreateFunc: func(event.CreateEvent) bool { + return false + }, + DeleteFunc: func(de event.DeleteEvent) bool { + return false + }, + GenericFunc: func(ge event.GenericEvent) bool { + return false + }, + })). + Complete(r) +} + +func (r *PodVolumeRestoreReconciler) findPVRForTargetPod(ctx context.Context, pod client.Object) []reconcile.Request { list := &velerov1api.PodVolumeRestoreList{} options := &client.ListOptions{ LabelSelector: labels.Set(map[string]string{ velerov1api.PodUIDLabel: string(pod.GetUID()), }).AsSelector(), } - if err := c.List(context.TODO(), list, options); err != nil { - c.logger.WithField("pod", fmt.Sprintf("%s/%s", pod.GetNamespace(), pod.GetName())).WithError(err). + if err := r.client.List(context.TODO(), list, options); err != nil { + r.logger.WithField("pod", fmt.Sprintf("%s/%s", pod.GetNamespace(), pod.GetName())).WithError(err). Error("unable to list PodVolumeRestores") return []reconcile.Request{} } - requests := make([]reconcile.Request, len(list.Items)) - for i, item := range list.Items { - requests[i] = reconcile.Request{ + + requests := []reconcile.Request{} + for _, item := range list.Items { + if isLegacyPVR(&item) { + continue + } + + requests = append(requests, reconcile.Request{ NamespacedName: types.NamespacedName{ Namespace: item.GetNamespace(), Name: item.GetName(), }, - } + }) } return requests } +func (r *PodVolumeRestoreReconciler) findPVRForRestorePod(ctx context.Context, podObj client.Object) []reconcile.Request { + pod := podObj.(*corev1api.Pod) + pvr, err := findPVRByRestorePod(r.client, *pod) + + log := r.logger.WithField("pod", pod.Name) + if err != nil { + log.WithError(err).Error("unable to get PVR") + return []reconcile.Request{} + } else if pvr == nil { + log.Error("get empty PVR") + return []reconcile.Request{} + } + log = log.WithFields(logrus.Fields{ + "PVR": pvr.Name, + }) + + if pvr.Status.Phase != velerov1api.PodVolumeRestorePhaseAccepted { + return []reconcile.Request{} + } + + if pod.Status.Phase == corev1api.PodRunning { + log.Info("Preparing PVR") + + if err = UpdatePVRWithRetry(context.Background(), r.client, types.NamespacedName{Namespace: pvr.Namespace, Name: pvr.Name}, log, + func(pvr *velerov1api.PodVolumeRestore) bool { + if isPVRInFinalState(pvr) { + log.Warnf("PVR %s is terminated, abort setting it to prepared", pvr.Name) + return false + } + + pvr.Status.Phase = velerov1api.PodVolumeRestorePhasePrepared + return true + }); err != nil { + log.WithError(err).Warn("failed to update PVR, prepare will halt for this PVR") + return []reconcile.Request{} + } + } else if unrecoverable, reason := kube.IsPodUnrecoverable(pod, log); unrecoverable { + err := UpdatePVRWithRetry(context.Background(), r.client, types.NamespacedName{Namespace: pvr.Namespace, Name: pvr.Name}, log, + func(pvr *velerov1api.PodVolumeRestore) bool { + if pvr.Spec.Cancel { + return false + } + + pvr.Spec.Cancel = true + pvr.Status.Message = fmt.Sprintf("Cancel PVR because the exposing pod %s/%s is in abnormal status for reason %s", pod.Namespace, pod.Name, reason) + + return true + }) + + if err != nil { + log.WithError(err).Warn("failed to cancel PVR, and it will wait for prepare timeout") + return []reconcile.Request{} + } + + log.Infof("Exposed pod is in abnormal status(reason %s) and PVR is marked as cancel", reason) + } else { + return []reconcile.Request{} + } + + request := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: pvr.Namespace, + Name: pvr.Name, + }, + } + return []reconcile.Request{request} +} + func isPVRNew(pvr *velerov1api.PodVolumeRestore) bool { return pvr.Status.Phase == "" || pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseNew } @@ -270,118 +725,186 @@ func getInitContainerIndex(pod *corev1api.Pod) int { return -1 } -func (c *PodVolumeRestoreReconciler) OnDataPathCompleted(ctx context.Context, namespace string, pvrName string, result datapath.Result) { - defer c.dataPathMgr.RemoveAsyncBR(pvrName) +func (r *PodVolumeRestoreReconciler) OnDataPathCompleted(ctx context.Context, namespace string, pvrName string, result datapath.Result) { + defer r.dataPathMgr.RemoveAsyncBR(pvrName) - log := c.logger.WithField("pvr", pvrName) + log := r.logger.WithField("PVR", pvrName) - log.WithField("PVR", pvrName).Info("Async fs restore data path completed") + log.WithField("PVR", pvrName).WithField("result", result.Restore).Info("Async fs restore data path completed") var pvr velerov1api.PodVolumeRestore - if err := c.Client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, &pvr); err != nil { + if err := r.client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, &pvr); err != nil { log.WithError(err).Warn("Failed to get PVR on completion") return } - volumePath := result.Restore.Target.ByPath - if volumePath == "" { - _, _ = c.errorOut(ctx, &pvr, errors.New("path is empty"), "invalid restore target", log) - return - } + log.Info("Cleaning up exposed environment") + r.exposer.CleanUp(ctx, getPVROwnerObject(&pvr)) - // Remove the .velero directory from the restored volume (it may contain done files from previous restores - // of this volume, which we don't want to carry over). If this fails for any reason, log and continue, since - // this is non-essential cleanup (the done files are named based on restore UID and the init container looks - // for the one specific to the restore being executed). - if err := os.RemoveAll(filepath.Join(volumePath, ".velero")); err != nil { - log.WithError(err).Warnf("error removing .velero directory from directory %s", volumePath) - } - - var restoreUID types.UID - for _, owner := range pvr.OwnerReferences { - if boolptr.IsSetToTrue(owner.Controller) { - restoreUID = owner.UID - break + if err := UpdatePVRWithRetry(ctx, r.client, types.NamespacedName{Namespace: pvr.Namespace, Name: pvr.Name}, log, func(pvr *velerov1api.PodVolumeRestore) bool { + if isPVRInFinalState(pvr) { + return false } - } - // Create the .velero directory within the volume dir so we can write a done file - // for this restore. - if err := os.MkdirAll(filepath.Join(volumePath, ".velero"), 0755); err != nil { - _, _ = c.errorOut(ctx, &pvr, err, "error creating .velero directory for done file", log) - return - } + pvr.Status.Phase = velerov1api.PodVolumeRestorePhaseCompleted + pvr.Status.CompletionTimestamp = &metav1.Time{Time: r.clock.Now()} - // Write a done file with name= into the just-created .velero dir - // within the volume. The velero init container on the pod is waiting - // for this file to exist in each restored volume before completing. - if err := os.WriteFile(filepath.Join(volumePath, ".velero", string(restoreUID)), nil, 0644); err != nil { //nolint:gosec // Internal usage. No need to check. - _, _ = c.errorOut(ctx, &pvr, err, "error writing done file", log) - return + return true + }); err != nil { + log.WithError(err).Error("error updating PVR status") + } else { + log.Info("Restore completed") } - - original := pvr.DeepCopy() - pvr.Status.Phase = velerov1api.PodVolumeRestorePhaseCompleted - pvr.Status.CompletionTimestamp = &metav1.Time{Time: c.clock.Now()} - if err := c.Patch(ctx, &pvr, client.MergeFrom(original)); err != nil { - log.WithError(err).Error("error updating PodVolumeRestore status") - } - - log.Info("Restore completed") } -func (c *PodVolumeRestoreReconciler) OnDataPathFailed(ctx context.Context, namespace string, pvrName string, err error) { - defer c.dataPathMgr.RemoveAsyncBR(pvrName) +func (r *PodVolumeRestoreReconciler) OnDataPathFailed(ctx context.Context, namespace string, pvrName string, err error) { + defer r.dataPathMgr.RemoveAsyncBR(pvrName) - log := c.logger.WithField("pvr", pvrName) + log := r.logger.WithField("PVR", pvrName) log.WithError(err).Error("Async fs restore data path failed") var pvr velerov1api.PodVolumeRestore - if getErr := c.Client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, &pvr); getErr != nil { + if getErr := r.client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, &pvr); getErr != nil { log.WithError(getErr).Warn("Failed to get PVR on failure") } else { - _, _ = c.errorOut(ctx, &pvr, err, "data path restore failed", log) + _, _ = r.errorOut(ctx, &pvr, err, "data path restore failed", log) } } -func (c *PodVolumeRestoreReconciler) OnDataPathCancelled(ctx context.Context, namespace string, pvrName string) { - defer c.dataPathMgr.RemoveAsyncBR(pvrName) +func (r *PodVolumeRestoreReconciler) OnDataPathCancelled(ctx context.Context, namespace string, pvrName string) { + defer r.dataPathMgr.RemoveAsyncBR(pvrName) - log := c.logger.WithField("pvr", pvrName) + log := r.logger.WithField("PVR", pvrName) log.Warn("Async fs restore data path canceled") var pvr velerov1api.PodVolumeRestore - if getErr := c.Client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, &pvr); getErr != nil { + if getErr := r.client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, &pvr); getErr != nil { log.WithError(getErr).Warn("Failed to get PVR on cancel") + return + } + // cleans up any objects generated during the snapshot expose + r.exposer.CleanUp(ctx, getPVROwnerObject(&pvr)) + + if err := UpdatePVRWithRetry(ctx, r.client, types.NamespacedName{Namespace: pvr.Namespace, Name: pvr.Name}, log, func(pvr *velerov1api.PodVolumeRestore) bool { + if isPVRInFinalState(pvr) { + return false + } + + pvr.Status.Phase = velerov1api.PodVolumeRestorePhaseCanceled + if pvr.Status.StartTimestamp.IsZero() { + pvr.Status.StartTimestamp = &metav1.Time{Time: r.clock.Now()} + } + pvr.Status.CompletionTimestamp = &metav1.Time{Time: r.clock.Now()} + + return true + }); err != nil { + log.WithError(err).Error("error updating PVR status on cancel") } else { - _, _ = c.errorOut(ctx, &pvr, errors.New("PVR is canceled"), "data path restore canceled", log) + delete(r.cancelledPVR, pvr.Name) } } -func (c *PodVolumeRestoreReconciler) OnDataPathProgress(ctx context.Context, namespace string, pvrName string, progress *uploader.Progress) { - log := c.logger.WithField("pvr", pvrName) +func (r *PodVolumeRestoreReconciler) OnDataPathProgress(ctx context.Context, namespace string, pvrName string, progress *uploader.Progress) { + log := r.logger.WithField("PVR", pvrName) - var pvr velerov1api.PodVolumeRestore - if err := c.Client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, &pvr); err != nil { - log.WithError(err).Warn("Failed to get PVB on progress") - return - } - - original := pvr.DeepCopy() - pvr.Status.Progress = veleroapishared.DataMoveOperationProgress{TotalBytes: progress.TotalBytes, BytesDone: progress.BytesDone} - - if err := c.Client.Patch(ctx, &pvr, client.MergeFrom(original)); err != nil { + if err := UpdatePVRWithRetry(ctx, r.client, types.NamespacedName{Namespace: namespace, Name: pvrName}, log, func(pvr *velerov1api.PodVolumeRestore) bool { + pvr.Status.Progress = veleroapishared.DataMoveOperationProgress{TotalBytes: progress.TotalBytes, BytesDone: progress.BytesDone} + return true + }); err != nil { log.WithError(err).Error("Failed to update progress") } } -func (c *PodVolumeRestoreReconciler) closeDataPath(ctx context.Context, pvbName string) { - fsRestore := c.dataPathMgr.GetAsyncBR(pvbName) - if fsRestore != nil { - fsRestore.Close(ctx) +func (r *PodVolumeRestoreReconciler) setupExposeParam(pvr *velerov1api.PodVolumeRestore) exposer.PodVolumeExposeParam { + log := r.logger.WithField("PVR", pvr.Name) + + hostingPodLabels := map[string]string{velerov1api.PVRLabel: pvr.Name} + for _, k := range util.ThirdPartyLabels { + if v, err := nodeagent.GetLabelValue(context.Background(), r.kubeClient, pvr.Namespace, k, ""); err != nil { + if err != nodeagent.ErrNodeAgentLabelNotFound { + log.WithError(err).Warnf("Failed to check node-agent label, skip adding host pod label %s", k) + } + } else { + hostingPodLabels[k] = v + } } - c.dataPathMgr.RemoveAsyncBR(pvbName) + hostingPodAnnotation := map[string]string{} + for _, k := range util.ThirdPartyAnnotations { + if v, err := nodeagent.GetAnnotationValue(context.Background(), r.kubeClient, pvr.Namespace, k, ""); err != nil { + if err != nodeagent.ErrNodeAgentAnnotationNotFound { + log.WithError(err).Warnf("Failed to check node-agent annotation, skip adding host pod annotation %s", k) + } + } else { + hostingPodAnnotation[k] = v + } + } + + return exposer.PodVolumeExposeParam{ + Type: exposer.PodVolumeExposeTypeRestore, + ClientNamespace: pvr.Spec.Pod.Namespace, + ClientPodName: pvr.Spec.Pod.Name, + ClientPodVolume: pvr.Spec.Volume, + HostingPodLabels: hostingPodLabels, + HostingPodAnnotations: hostingPodAnnotation, + OperationTimeout: r.resourceTimeout, + Resources: r.podResources, + } +} + +func getPVROwnerObject(pvr *velerov1api.PodVolumeRestore) corev1api.ObjectReference { + return corev1api.ObjectReference{ + Kind: pvr.Kind, + Namespace: pvr.Namespace, + Name: pvr.Name, + UID: pvr.UID, + APIVersion: pvr.APIVersion, + } +} + +func findPVRByRestorePod(client client.Client, pod corev1api.Pod) (*velerov1api.PodVolumeRestore, error) { + if label, exist := pod.Labels[velerov1api.PVRLabel]; exist { + pvr := &velerov1api.PodVolumeRestore{} + err := client.Get(context.Background(), types.NamespacedName{ + Namespace: pod.Namespace, + Name: label, + }, pvr) + + if err != nil { + return nil, errors.Wrapf(err, "error to find PVR by pod %s/%s", pod.Namespace, pod.Name) + } + return pvr, nil + } + return nil, nil +} + +func isPVRInFinalState(pvr *velerov1api.PodVolumeRestore) bool { + return pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseFailed || + pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseCanceled || + pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseCompleted +} + +func UpdatePVRWithRetry(ctx context.Context, client client.Client, namespacedName types.NamespacedName, log logrus.FieldLogger, updateFunc func(*velerov1api.PodVolumeRestore) bool) error { + return wait.PollUntilContextCancel(ctx, time.Millisecond*100, true, func(ctx context.Context) (bool, error) { + pvr := &velerov1api.PodVolumeRestore{} + if err := client.Get(ctx, namespacedName, pvr); err != nil { + return false, errors.Wrap(err, "getting PVR") + } + + if updateFunc(pvr) { + err := client.Update(ctx, pvr) + if err != nil { + if apierrors.IsConflict(err) { + log.Warnf("failed to update PVR for %s/%s and will retry it", pvr.Namespace, pvr.Name) + return false, nil + } else { + return false, errors.Wrapf(err, "error updating PVR %s/%s", pvr.Namespace, pvr.Name) + } + } + } + + return true, nil + }) } diff --git a/pkg/controller/pod_volume_restore_controller_legacy.go b/pkg/controller/pod_volume_restore_controller_legacy.go new file mode 100644 index 000000000..03c8b4c56 --- /dev/null +++ b/pkg/controller/pod_volume_restore_controller_legacy.go @@ -0,0 +1,10 @@ +package controller + +import ( + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/uploader" +) + +func isLegacyPVR(pvr *velerov1api.PodVolumeRestore) bool { + return pvr.Spec.UploaderType == uploader.ResticType +} diff --git a/pkg/controller/pod_volume_restore_controller_test.go b/pkg/controller/pod_volume_restore_controller_test.go index f24aed653..5d4a3522e 100644 --- a/pkg/controller/pod_volume_restore_controller_test.go +++ b/pkg/controller/pod_volume_restore_controller_test.go @@ -18,21 +18,43 @@ package controller import ( "context" + "fmt" "testing" "time" + "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + clientgofake "k8s.io/client-go/kubernetes/fake" clocks "k8s.io/utils/clock" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/builder" + "github.com/vmware-tanzu/velero/pkg/datapath" + datapathmockes "github.com/vmware-tanzu/velero/pkg/datapath/mocks" + "github.com/vmware-tanzu/velero/pkg/exposer" + exposermockes "github.com/vmware-tanzu/velero/pkg/exposer/mocks" "github.com/vmware-tanzu/velero/pkg/restorehelper" "github.com/vmware-tanzu/velero/pkg/test" + velerotest "github.com/vmware-tanzu/velero/pkg/test" + "github.com/vmware-tanzu/velero/pkg/uploader" + "github.com/vmware-tanzu/velero/pkg/util/boolptr" + "github.com/vmware-tanzu/velero/pkg/util/kube" ) func TestShouldProcess(t *testing.T) { @@ -195,11 +217,11 @@ func TestShouldProcess(t *testing.T) { c := &PodVolumeRestoreReconciler{ logger: logrus.New(), - Client: cli, + client: cli, clock: &clocks.RealClock{}, } - shouldProcess, _, _ := c.shouldProcess(ctx, c.logger, ts.obj) + shouldProcess, _, _ := shouldProcess(ctx, c.client, c.logger, ts.obj) require.Equal(t, ts.shouldProcessed, shouldProcess) }) } @@ -478,7 +500,7 @@ func TestGetInitContainerIndex(t *testing.T) { } } -func TestFindVolumeRestoresForPod(t *testing.T) { +func TestFindPVRForTargetPod(t *testing.T) { pod := &corev1api.Pod{} pod.UID = "uid" @@ -488,14 +510,14 @@ func TestFindVolumeRestoresForPod(t *testing.T) { // no matching PVR reconciler := &PodVolumeRestoreReconciler{ - Client: clientBuilder.Build(), + client: clientBuilder.Build(), logger: logrus.New(), } - requests := reconciler.findVolumeRestoresForPod(context.Background(), pod) + requests := reconciler.findPVRForTargetPod(context.Background(), pod) assert.Empty(t, requests) // contain one matching PVR - reconciler.Client = clientBuilder.WithLists(&velerov1api.PodVolumeRestoreList{ + reconciler.client = clientBuilder.WithLists(&velerov1api.PodVolumeRestoreList{ Items: []velerov1api.PodVolumeRestore{ { ObjectMeta: metav1.ObjectMeta{ @@ -515,6 +537,902 @@ func TestFindVolumeRestoresForPod(t *testing.T) { }, }, }).Build() - requests = reconciler.findVolumeRestoresForPod(context.Background(), pod) + requests = reconciler.findPVRForTargetPod(context.Background(), pod) assert.Len(t, requests, 1) } + +const pvrName string = "pvr-1" + +func pvrBuilder() *builder.PodVolumeRestoreBuilder { + return builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName). + BackupStorageLocation("bsl-loc"). + SnapshotID("test-snapshot-id") +} + +func initPodVolumeRestoreReconciler(objects []runtime.Object, cliObj []client.Object, needError ...bool) (*PodVolumeRestoreReconciler, error) { + var errs = make([]error, 6) + for k, isError := range needError { + if k == 0 && isError { + errs[0] = fmt.Errorf("Get error") + } else if k == 1 && isError { + errs[1] = fmt.Errorf("Create error") + } else if k == 2 && isError { + errs[2] = fmt.Errorf("Update error") + } else if k == 3 && isError { + errs[3] = fmt.Errorf("Patch error") + } else if k == 4 && isError { + errs[4] = apierrors.NewConflict(velerov1api.Resource("podvolumerestore"), pvrName, errors.New("conflict")) + } else if k == 5 && isError { + errs[5] = fmt.Errorf("List error") + } + } + return initPodVolumeRestoreReconcilerWithError(objects, cliObj, errs...) +} + +func initPodVolumeRestoreReconcilerWithError(objects []runtime.Object, cliObj []client.Object, needError ...error) (*PodVolumeRestoreReconciler, error) { + scheme := runtime.NewScheme() + err := velerov1api.AddToScheme(scheme) + if err != nil { + return nil, err + } + + err = corev1api.AddToScheme(scheme) + if err != nil { + return nil, err + } + + fakeClient := &FakeClient{ + Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(cliObj...).Build(), + } + + for k := range needError { + if k == 0 { + fakeClient.getError = needError[0] + } else if k == 1 { + fakeClient.createError = needError[1] + } else if k == 2 { + fakeClient.updateError = needError[2] + } else if k == 3 { + fakeClient.patchError = needError[3] + } else if k == 4 { + fakeClient.updateConflict = needError[4] + } else if k == 5 { + fakeClient.listError = needError[5] + } + } + + var fakeKubeClient *clientgofake.Clientset + if len(objects) != 0 { + fakeKubeClient = clientgofake.NewSimpleClientset(objects...) + } else { + fakeKubeClient = clientgofake.NewSimpleClientset() + } + + fakeFS := velerotest.NewFakeFileSystem() + pathGlob := fmt.Sprintf("/host_pods/%s/volumes/*/%s", "test-uid", "test-pvc") + _, err = fakeFS.Create(pathGlob) + if err != nil { + return nil, err + } + + dataPathMgr := datapath.NewManager(1) + + return NewPodVolumeRestoreReconciler(fakeClient, nil, fakeKubeClient, dataPathMgr, "test-node", time.Minute*5, time.Minute, corev1api.ResourceRequirements{}, velerotest.NewLogger()), nil +} + +func TestPodVolumeRestoreReconcile(t *testing.T) { + daemonSet := &appsv1api.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "velero", + Name: "node-agent", + }, + TypeMeta: metav1.TypeMeta{ + Kind: "DaemonSet", + APIVersion: appsv1api.SchemeGroupVersion.String(), + }, + Spec: appsv1api.DaemonSetSpec{ + Template: corev1api.PodTemplateSpec{ + Spec: corev1api.PodSpec{ + Containers: []corev1api.Container{ + { + Image: "fake-image", + }, + }, + }, + }, + }, + } + + node := builder.ForNode("fake-node").Labels(map[string]string{kube.NodeOSLabel: kube.NodeOSLinux}).Result() + + tests := []struct { + name string + pvr *velerov1api.PodVolumeRestore + notCreatePVR bool + targetPod *corev1api.Pod + dataMgr *datapath.Manager + needErrs []bool + needCreateFSBR bool + needDelete bool + sportTime *metav1.Time + mockExposeErr *bool + isGetExposeErr bool + isGetExposeNil bool + isPeekExposeErr bool + isNilExposer bool + notNilExpose bool + notMockCleanUp bool + mockInit bool + mockInitErr error + mockStart bool + mockStartErr error + mockCancel bool + mockClose bool + needExclusiveUpdateError error + expected *velerov1api.PodVolumeRestore + expectDeleted bool + expectCancelRecord bool + expectedResult *ctrl.Result + expectedErr string + expectDataPath bool + }{ + { + name: "pvr not found", + pvr: pvrBuilder().Result(), + notCreatePVR: true, + }, + { + name: "pvr not created in velero default namespace", + pvr: builder.ForPodVolumeRestore("test-ns", pvrName).Result(), + }, + { + name: "get dd fail", + pvr: builder.ForPodVolumeRestore("test-ns", pvrName).Result(), + needErrs: []bool{true, false, false, false}, + expectedErr: "Get error", + }, + { + name: "add finalizer to pvr", + pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Result(), + expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Result(), + }, + { + name: "add finalizer to pvr failed", + pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Result(), + needErrs: []bool{false, false, true, false}, + expectedErr: "error updating PVR velero/pvr-1: Update error", + }, + { + name: "pvr is under deletion", + pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Result(), + needDelete: true, + expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Result(), + }, + { + name: "pvr is under deletion but cancel failed", + pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Result(), + needErrs: []bool{false, false, true, false}, + needDelete: true, + expectedErr: "error updating PVR velero/pvr-1: Update error", + }, + { + name: "pvr is under deletion and in terminal state", + pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Phase(velerov1api.PodVolumeRestorePhaseFailed).Result(), + sportTime: &metav1.Time{Time: time.Now()}, + needDelete: true, + expectDeleted: true, + }, + { + name: "pvr is under deletion and in terminal state, but remove finalizer failed", + pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Phase(velerov1api.PodVolumeRestorePhaseFailed).Result(), + needErrs: []bool{false, false, true, false}, + needDelete: true, + expectedErr: "error updating PVR velero/pvr-1: Update error", + }, + { + name: "delay cancel negative for others", + pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeRestorePhasePrepared).Result(), + sportTime: &metav1.Time{Time: time.Now()}, + expectCancelRecord: true, + }, + { + name: "delay cancel negative for inProgress", + pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Result(), + sportTime: &metav1.Time{Time: time.Now().Add(-time.Minute * 58)}, + expectCancelRecord: true, + }, + { + name: "delay cancel affirmative for others", + pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeRestorePhasePrepared).Result(), + sportTime: &metav1.Time{Time: time.Now().Add(-time.Minute * 5)}, + expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeRestorePhaseCanceled).Result(), + }, + { + name: "delay cancel affirmative for inProgress", + pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Result(), + sportTime: &metav1.Time{Time: time.Now().Add(-time.Hour)}, + expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeRestorePhaseCanceled).Result(), + }, + { + name: "delay cancel failed", + pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Result(), + needErrs: []bool{false, false, true, false}, + sportTime: &metav1.Time{Time: time.Now().Add(-time.Hour)}, + expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Result(), + expectCancelRecord: true, + }, + { + name: "Unknown pvr status", + pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase("Unknown").Finalizers([]string{PodVolumeFinalizer}).Result(), + }, + { + name: "new pvr but accept failed", + pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).PodNamespace("test-ns").PodName("test-pod").Result(), + targetPod: builder.ForPod("test-ns", "test-pod").InitContainers(&corev1api.Container{Name: restorehelper.WaitInitContainer}).InitContainerState(corev1api.ContainerState{Running: &corev1api.ContainerStateRunning{}}).Result(), + needErrs: []bool{false, false, true, false}, + expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Result(), + expectedErr: "error accepting PVR pvr-1: error updating PVR velero/pvr-1: Update error", + }, + { + name: "pvr is cancel on accepted", + pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Result(), + expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeRestorePhaseCanceled).Result(), + }, + { + name: "pvr expose failed", + pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).PodNamespace("test-ns").PodName("test-pod").Finalizers([]string{PodVolumeFinalizer}).Result(), + targetPod: builder.ForPod("test-ns", "test-pod").InitContainers(&corev1api.Container{Name: restorehelper.WaitInitContainer}).InitContainerState(corev1api.ContainerState{Running: &corev1api.ContainerStateRunning{}}).Result(), + mockExposeErr: boolptr.True(), + expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Phase(velerov1api.PodVolumeRestorePhaseFailed).Message("error to expose PVR").Result(), + expectedErr: "Error to expose restore exposer", + }, + { + name: "pvr succeeds for accepted", + pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).PodNamespace("test-ns").PodName("test-pod").Finalizers([]string{PodVolumeFinalizer}).Result(), + mockExposeErr: boolptr.False(), + notMockCleanUp: true, + targetPod: builder.ForPod("test-ns", "test-pod").InitContainers(&corev1api.Container{Name: restorehelper.WaitInitContainer}).InitContainerState(corev1api.ContainerState{Running: &corev1api.ContainerStateRunning{}}).Result(), + expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Finalizers([]string{PodVolumeFinalizer}).Phase(velerov1api.PodVolumeRestorePhaseAccepted).Result(), + }, + { + name: "prepare timeout on accepted", + pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseAccepted).Finalizers([]string{PodVolumeFinalizer}).AcceptedTimestamp(&metav1.Time{Time: time.Now().Add(-time.Minute * 30)}).Result(), + expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseFailed).Finalizers([]string{PodVolumeFinalizer}).Phase(velerov1api.PodVolumeRestorePhaseFailed).Message("timeout on preparing PVR").Result(), + }, + { + name: "peek error on accepted", + pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseAccepted).Finalizers([]string{PodVolumeFinalizer}).Result(), + isPeekExposeErr: true, + expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseCanceled).Finalizers([]string{PodVolumeFinalizer}).Phase(velerov1api.PodVolumeRestorePhaseCanceled).Message("found a PVR velero/pvr-1 with expose error: fake-peek-error. mark it as cancel").Result(), + }, + { + name: "cancel on pvr", + pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Node("test-node").Result(), + expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseCanceled).Finalizers([]string{PodVolumeFinalizer}).Cancel(true).Phase(velerov1api.PodVolumeRestorePhaseCanceled).Result(), + }, + { + name: "Failed to get restore expose on prepared", + pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), + isGetExposeErr: true, + expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseFailed).Finalizers([]string{PodVolumeFinalizer}).Message("exposed PVR is not ready").Result(), + expectedErr: "Error to get PVR exposer", + }, + { + name: "Get nil restore expose on prepared", + pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), + isGetExposeNil: true, + expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseFailed).Finalizers([]string{PodVolumeFinalizer}).Message("exposed PVR is not ready").Result(), + expectedErr: "no expose result is available for the current node", + }, + { + name: "Error in data path is concurrent limited", + pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), + dataMgr: datapath.NewManager(0), + notNilExpose: true, + notMockCleanUp: true, + expectedResult: &ctrl.Result{Requeue: true, RequeueAfter: time.Second * 5}, + }, + { + name: "data path init error", + pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), + mockInit: true, + mockInitErr: errors.New("fake-data-path-init-error"), + mockClose: true, + notNilExpose: true, + expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseFailed).Finalizers([]string{PodVolumeFinalizer}).Message("error initializing data path").Result(), + expectedErr: "error initializing asyncBR: fake-data-path-init-error", + }, + { + name: "Unable to update status to in progress for pvr", + pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), + needErrs: []bool{false, false, true, false}, + mockInit: true, + mockClose: true, + notNilExpose: true, + notMockCleanUp: true, + expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Result(), + }, + { + name: "data path start error", + pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), + mockInit: true, + mockStart: true, + mockStartErr: errors.New("fake-data-path-start-error"), + mockClose: true, + notNilExpose: true, + expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseFailed).Finalizers([]string{PodVolumeFinalizer}).Message("error starting data path").Result(), + expectedErr: "error starting async restore for pod test-name, volume test-pvc: fake-data-path-start-error", + }, + { + name: "Prepare succeeds", + pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhasePrepared).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), + mockInit: true, + mockStart: true, + notNilExpose: true, + notMockCleanUp: true, + expectDataPath: true, + expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Finalizers([]string{PodVolumeFinalizer}).Result(), + }, + { + name: "In progress pvr is not handled by the current node", + pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Finalizers([]string{PodVolumeFinalizer}).Result(), + expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Finalizers([]string{PodVolumeFinalizer}).Result(), + }, + { + name: "In progress pvr is not set as cancel", + pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), + expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Finalizers([]string{PodVolumeFinalizer}).Result(), + }, + { + name: "Cancel pvr in progress with empty FSBR", + pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Cancel(true).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), + expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseCanceled).Cancel(true).Finalizers([]string{PodVolumeFinalizer}).Result(), + }, + { + name: "Cancel pvr in progress and patch pvr error", + pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Cancel(true).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), + needErrs: []bool{false, false, true, false}, + needCreateFSBR: true, + expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Cancel(true).Finalizers([]string{PodVolumeFinalizer}).Result(), + expectedErr: "error updating PVR velero/pvr-1: Update error", + expectCancelRecord: true, + expectDataPath: true, + }, + { + name: "Cancel pvr in progress succeeds", + pvr: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Cancel(true).Finalizers([]string{PodVolumeFinalizer}).Node("test-node").Result(), + needCreateFSBR: true, + mockCancel: true, + expected: builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Phase(velerov1api.PodVolumeRestorePhaseCanceling).Cancel(true).Finalizers([]string{PodVolumeFinalizer}).Result(), + expectDataPath: true, + expectCancelRecord: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + objs := []runtime.Object{daemonSet, node} + + ctlObj := []client.Object{} + if test.targetPod != nil { + ctlObj = append(ctlObj, test.targetPod) + } + + r, err := initPodVolumeRestoreReconciler(objs, ctlObj, test.needErrs...) + require.NoError(t, err) + + if !test.notCreatePVR { + err = r.client.Create(context.Background(), test.pvr) + require.NoError(t, err) + } + + if test.needDelete { + err = r.client.Delete(context.Background(), test.pvr) + require.NoError(t, err) + } + + if test.dataMgr != nil { + r.dataPathMgr = test.dataMgr + } else { + r.dataPathMgr = datapath.NewManager(1) + } + + if test.sportTime != nil { + r.cancelledPVR[test.pvr.Name] = test.sportTime.Time + } + + funcExclusiveUpdatePodVolumeRestore = exclusiveUpdatePodVolumeRestore + if test.needExclusiveUpdateError != nil { + funcExclusiveUpdatePodVolumeRestore = func(context.Context, kbclient.Client, *velerov1api.PodVolumeRestore, func(*velerov1api.PodVolumeRestore)) (bool, error) { + return false, test.needExclusiveUpdateError + } + } + + datapath.MicroServiceBRWatcherCreator = func(kbclient.Client, kubernetes.Interface, manager.Manager, string, string, + string, string, string, string, datapath.Callbacks, logrus.FieldLogger) datapath.AsyncBR { + asyncBR := datapathmockes.NewAsyncBR(t) + if test.mockInit { + asyncBR.On("Init", mock.Anything, mock.Anything).Return(test.mockInitErr) + } + + if test.mockStart { + asyncBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything).Return(test.mockStartErr) + } + + if test.mockCancel { + asyncBR.On("Cancel").Return() + } + + if test.mockClose { + asyncBR.On("Close", mock.Anything).Return() + } + + return asyncBR + } + + if test.mockExposeErr != nil || test.isGetExposeErr || test.isGetExposeNil || test.isPeekExposeErr || test.isNilExposer || test.notNilExpose { + if test.isNilExposer { + r.exposer = nil + } else { + r.exposer = func() exposer.PodVolumeExposer { + ep := exposermockes.NewPodVolumeExposer(t) + if test.mockExposeErr != nil { + if boolptr.IsSetToTrue(test.mockExposeErr) { + ep.On("Expose", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(errors.New("Error to expose restore exposer")) + } else { + ep.On("Expose", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + } + } else if test.notNilExpose { + hostingPod := builder.ForPod("test-ns", "test-name").Volumes(&corev1api.Volume{Name: "test-pvc"}).Result() + hostingPod.ObjectMeta.SetUID("test-uid") + ep.On("GetExposed", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&exposer.ExposeResult{ByPod: exposer.ExposeByPod{HostingPod: hostingPod, VolumeName: "test-pvc"}}, nil) + } else if test.isGetExposeErr { + ep.On("GetExposed", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, errors.New("Error to get PVR exposer")) + } else if test.isGetExposeNil { + ep.On("GetExposed", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, nil) + } else if test.isPeekExposeErr { + ep.On("PeekExposed", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(errors.New("fake-peek-error")) + } + + if !test.notMockCleanUp { + ep.On("CleanUp", mock.Anything, mock.Anything).Return() + } + return ep + }() + } + } + + if test.needCreateFSBR { + if fsBR := r.dataPathMgr.GetAsyncBR(test.pvr.Name); fsBR == nil { + _, err := r.dataPathMgr.CreateMicroServiceBRWatcher(ctx, r.client, nil, nil, datapath.TaskTypeRestore, test.pvr.Name, pVBRRequestor, + velerov1api.DefaultNamespace, "", "", datapath.Callbacks{OnCancelled: r.OnDataPathCancelled}, false, velerotest.NewLogger()) + require.NoError(t, err) + } + } + + actualResult, err := r.Reconcile(ctx, ctrl.Request{ + NamespacedName: types.NamespacedName{ + Namespace: velerov1api.DefaultNamespace, + Name: test.pvr.Name, + }, + }) + + if test.expectedErr != "" { + assert.EqualError(t, err, test.expectedErr) + } else { + assert.NoError(t, err) + } + + if test.expectedResult != nil { + assert.Equal(t, test.expectedResult.Requeue, actualResult.Requeue) + assert.Equal(t, test.expectedResult.RequeueAfter, actualResult.RequeueAfter) + } + + if test.expected != nil || test.expectDeleted { + pvr := velerov1api.PodVolumeRestore{} + err = r.client.Get(ctx, kbclient.ObjectKey{ + Name: test.pvr.Name, + Namespace: test.pvr.Namespace, + }, &pvr) + + if test.expectDeleted { + assert.True(t, apierrors.IsNotFound(err)) + } else { + require.NoError(t, err) + + assert.Equal(t, test.expected.Status.Phase, pvr.Status.Phase) + assert.Contains(t, pvr.Status.Message, test.expected.Status.Message) + assert.Equal(t, test.expected.Finalizers, pvr.Finalizers) + assert.Equal(t, test.expected.Spec.Cancel, pvr.Spec.Cancel) + } + } + + if !test.expectDataPath { + assert.Nil(t, r.dataPathMgr.GetAsyncBR(test.pvr.Name)) + } else { + assert.NotNil(t, r.dataPathMgr.GetAsyncBR(test.pvr.Name)) + } + + if test.expectCancelRecord { + assert.Contains(t, r.cancelledPVR, test.pvr.Name) + } else { + assert.Empty(t, r.cancelledPVR) + } + }) + } +} + +func TestOnPodVolumeRestoreFailed(t *testing.T) { + for _, getErr := range []bool{true, false} { + ctx := context.TODO() + needErrs := []bool{getErr, false, false, false} + r, err := initPodVolumeRestoreReconciler(nil, []client.Object{}, needErrs...) + require.NoError(t, err) + + pvr := pvrBuilder().Result() + namespace := pvr.Namespace + pvrName := pvr.Name + + assert.NoError(t, r.client.Create(ctx, pvr)) + r.OnDataPathFailed(ctx, namespace, pvrName, fmt.Errorf("Failed to handle %v", pvrName)) + updatedPVR := &velerov1api.PodVolumeRestore{} + if getErr { + assert.Error(t, r.client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, updatedPVR)) + assert.NotEqual(t, velerov1api.PodVolumeRestorePhaseFailed, updatedPVR.Status.Phase) + assert.True(t, updatedPVR.Status.StartTimestamp.IsZero()) + } else { + assert.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, updatedPVR)) + assert.Equal(t, velerov1api.PodVolumeRestorePhaseFailed, updatedPVR.Status.Phase) + assert.True(t, updatedPVR.Status.StartTimestamp.IsZero()) + } + } +} + +func TestOnPodVolumeRestoreCancelled(t *testing.T) { + for _, getErr := range []bool{true, false} { + ctx := context.TODO() + needErrs := []bool{getErr, false, false, false} + r, err := initPodVolumeRestoreReconciler(nil, nil, needErrs...) + require.NoError(t, err) + + pvr := pvrBuilder().Result() + namespace := pvr.Namespace + pvrName := pvr.Name + + assert.NoError(t, r.client.Create(ctx, pvr)) + r.OnDataPathCancelled(ctx, namespace, pvrName) + updatedPVR := &velerov1api.PodVolumeRestore{} + if getErr { + assert.Error(t, r.client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, updatedPVR)) + assert.NotEqual(t, velerov1api.PodVolumeRestorePhaseFailed, updatedPVR.Status.Phase) + assert.True(t, updatedPVR.Status.StartTimestamp.IsZero()) + } else { + assert.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, updatedPVR)) + assert.Equal(t, velerov1api.PodVolumeRestorePhaseCanceled, updatedPVR.Status.Phase) + assert.False(t, updatedPVR.Status.StartTimestamp.IsZero()) + assert.False(t, updatedPVR.Status.CompletionTimestamp.IsZero()) + } + } +} + +func TestOnPodVolumeRestoreCompleted(t *testing.T) { + tests := []struct { + name string + emptyFSBR bool + isGetErr bool + rebindVolumeErr bool + }{ + { + name: "PVR complete", + emptyFSBR: false, + isGetErr: false, + rebindVolumeErr: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctx := context.TODO() + needErrs := []bool{test.isGetErr, false, false, false} + r, err := initPodVolumeRestoreReconciler(nil, []client.Object{}, needErrs...) + r.exposer = func() exposer.PodVolumeExposer { + ep := exposermockes.NewPodVolumeExposer(t) + ep.On("CleanUp", mock.Anything, mock.Anything).Return() + return ep + }() + + require.NoError(t, err) + pvr := builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, pvrName).Result() + namespace := pvr.Namespace + ddName := pvr.Name + + assert.NoError(t, r.client.Create(ctx, pvr)) + r.OnDataPathCompleted(ctx, namespace, ddName, datapath.Result{}) + updatedDD := &velerov1api.PodVolumeRestore{} + if test.isGetErr { + assert.Error(t, r.client.Get(ctx, types.NamespacedName{Name: ddName, Namespace: namespace}, updatedDD)) + assert.Equal(t, velerov1api.PodVolumeRestorePhase(""), updatedDD.Status.Phase) + assert.True(t, updatedDD.Status.CompletionTimestamp.IsZero()) + } else { + assert.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: ddName, Namespace: namespace}, updatedDD)) + assert.Equal(t, velerov1api.PodVolumeRestorePhaseCompleted, updatedDD.Status.Phase) + assert.False(t, updatedDD.Status.CompletionTimestamp.IsZero()) + } + }) + } +} + +func TestOnPodVolumeRestoreProgress(t *testing.T) { + totalBytes := int64(1024) + bytesDone := int64(512) + tests := []struct { + name string + pvr *velerov1api.PodVolumeRestore + progress uploader.Progress + needErrs []bool + }{ + { + name: "patch in progress phase success", + pvr: pvrBuilder().Result(), + progress: uploader.Progress{ + TotalBytes: totalBytes, + BytesDone: bytesDone, + }, + }, + { + name: "failed to get pvr", + pvr: pvrBuilder().Result(), + needErrs: []bool{true, false, false, false}, + }, + { + name: "failed to patch pvr", + pvr: pvrBuilder().Result(), + needErrs: []bool{false, false, true, false}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctx := context.TODO() + + r, err := initPodVolumeRestoreReconciler(nil, []client.Object{}, test.needErrs...) + require.NoError(t, err) + defer func() { + r.client.Delete(ctx, test.pvr, &kbclient.DeleteOptions{}) + }() + + pvr := pvrBuilder().Result() + namespace := pvr.Namespace + pvrName := pvr.Name + + assert.NoError(t, r.client.Create(context.Background(), pvr)) + + // Create a Progress object + progress := &uploader.Progress{ + TotalBytes: totalBytes, + BytesDone: bytesDone, + } + + r.OnDataPathProgress(ctx, namespace, pvrName, progress) + if len(test.needErrs) != 0 && !test.needErrs[0] { + updatedPVR := &velerov1api.PodVolumeRestore{} + assert.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, updatedPVR)) + assert.Equal(t, test.progress.TotalBytes, updatedPVR.Status.Progress.TotalBytes) + assert.Equal(t, test.progress.BytesDone, updatedPVR.Status.Progress.BytesDone) + } + }) + } +} + +func TestFindPVBForRestorePod(t *testing.T) { + needErrs := []bool{false, false, false, false} + r, err := initPodVolumeRestoreReconciler(nil, []client.Object{}, needErrs...) + require.NoError(t, err) + tests := []struct { + name string + pvr *velerov1api.PodVolumeRestore + pod *corev1api.Pod + checkFunc func(*velerov1api.PodVolumeRestore, []reconcile.Request) + }{ + { + name: "find pvr for pod", + pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseAccepted).Result(), + pod: builder.ForPod(velerov1api.DefaultNamespace, pvrName).Labels(map[string]string{velerov1api.PVRLabel: pvrName}).Status(corev1api.PodStatus{Phase: corev1api.PodRunning}).Result(), + checkFunc: func(pvr *velerov1api.PodVolumeRestore, requests []reconcile.Request) { + // Assert that the function returns a single request + assert.Len(t, requests, 1) + // Assert that the request contains the correct namespaced name + assert.Equal(t, pvr.Namespace, requests[0].Namespace) + assert.Equal(t, pvr.Name, requests[0].Name) + }, + }, { + name: "no selected label found for pod", + pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseAccepted).Result(), + pod: builder.ForPod(velerov1api.DefaultNamespace, pvrName).Result(), + checkFunc: func(pvr *velerov1api.PodVolumeRestore, requests []reconcile.Request) { + // Assert that the function returns a single request + assert.Empty(t, requests) + }, + }, { + name: "no matched pod", + pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseAccepted).Result(), + pod: builder.ForPod(velerov1api.DefaultNamespace, pvrName).Labels(map[string]string{velerov1api.PVRLabel: "non-existing-pvr"}).Result(), + checkFunc: func(pvr *velerov1api.PodVolumeRestore, requests []reconcile.Request) { + assert.Empty(t, requests) + }, + }, + { + name: "pvr not accept", + pvr: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseInProgress).Result(), + pod: builder.ForPod(velerov1api.DefaultNamespace, pvrName).Labels(map[string]string{velerov1api.PVRLabel: pvrName}).Result(), + checkFunc: func(pvr *velerov1api.PodVolumeRestore, requests []reconcile.Request) { + assert.Empty(t, requests) + }, + }, + } + for _, test := range tests { + ctx := context.Background() + assert.NoError(t, r.client.Create(ctx, test.pod)) + assert.NoError(t, r.client.Create(ctx, test.pvr)) + // Call the findSnapshotRestoreForPod function + requests := r.findPVRForRestorePod(context.Background(), test.pod) + test.checkFunc(test.pvr, requests) + r.client.Delete(ctx, test.pvr, &kbclient.DeleteOptions{}) + if test.pod != nil { + r.client.Delete(ctx, test.pod, &kbclient.DeleteOptions{}) + } + } +} + +func TestOnPVRPrepareTimeout(t *testing.T) { + tests := []struct { + name string + pvr *velerov1api.PodVolumeRestore + needErrs []error + expected *velerov1api.PodVolumeRestore + }{ + { + name: "update fail", + pvr: pvrBuilder().Result(), + needErrs: []error{nil, nil, fmt.Errorf("fake-update-error"), nil}, + expected: pvrBuilder().Result(), + }, + { + name: "update interrupted", + pvr: pvrBuilder().Result(), + needErrs: []error{nil, nil, &fakeAPIStatus{metav1.StatusReasonConflict}, nil}, + expected: pvrBuilder().Result(), + }, + { + name: "succeed", + pvr: pvrBuilder().Result(), + needErrs: []error{nil, nil, nil, nil}, + expected: pvrBuilder().Phase(velerov1api.PodVolumeRestorePhaseFailed).Result(), + }, + } + for _, test := range tests { + ctx := context.Background() + r, err := initPodVolumeRestoreReconcilerWithError(nil, []client.Object{}, test.needErrs...) + require.NoError(t, err) + + err = r.client.Create(ctx, test.pvr) + require.NoError(t, err) + + r.onPrepareTimeout(ctx, test.pvr) + + pvr := velerov1api.PodVolumeRestore{} + _ = r.client.Get(ctx, kbclient.ObjectKey{ + Name: test.pvr.Name, + Namespace: test.pvr.Namespace, + }, &pvr) + + assert.Equal(t, test.expected.Status.Phase, pvr.Status.Phase) + } +} + +func TestTryCancelPVR(t *testing.T) { + tests := []struct { + name string + pvr *velerov1api.PodVolumeRestore + needErrs []error + succeeded bool + expectedErr string + }{ + { + name: "update fail", + pvr: pvrBuilder().Result(), + needErrs: []error{nil, nil, fmt.Errorf("fake-update-error"), nil}, + }, + { + name: "cancel by others", + pvr: pvrBuilder().Result(), + needErrs: []error{nil, nil, &fakeAPIStatus{metav1.StatusReasonConflict}, nil}, + }, + { + name: "succeed", + pvr: pvrBuilder().Result(), + needErrs: []error{nil, nil, nil, nil}, + succeeded: true, + }, + } + for _, test := range tests { + ctx := context.Background() + r, err := initPodVolumeRestoreReconcilerWithError(nil, []client.Object{}, test.needErrs...) + require.NoError(t, err) + + err = r.client.Create(ctx, test.pvr) + require.NoError(t, err) + + r.tryCancelPodVolumeRestore(ctx, test.pvr, "") + + if test.expectedErr == "" { + assert.NoError(t, err) + } else { + assert.EqualError(t, err, test.expectedErr) + } + } +} + +func TestUpdatePVRWithRetry(t *testing.T) { + namespacedName := types.NamespacedName{ + Name: pvrName, + Namespace: "velero", + } + + // Define test cases + testCases := []struct { + Name string + needErrs []bool + noChange bool + ExpectErr bool + }{ + { + Name: "SuccessOnFirstAttempt", + }, + { + Name: "Error get", + needErrs: []bool{true, false, false, false, false}, + ExpectErr: true, + }, + { + Name: "Error update", + needErrs: []bool{false, false, true, false, false}, + ExpectErr: true, + }, + { + Name: "no change", + noChange: true, + needErrs: []bool{false, false, true, false, false}, + }, + { + Name: "Conflict with error timeout", + needErrs: []bool{false, false, false, false, true}, + ExpectErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.Name, func(t *testing.T) { + ctx, cancelFunc := context.WithTimeout(context.TODO(), time.Second*5) + defer cancelFunc() + r, err := initPodVolumeRestoreReconciler(nil, []client.Object{}, tc.needErrs...) + require.NoError(t, err) + err = r.client.Create(ctx, pvrBuilder().Result()) + require.NoError(t, err) + updateFunc := func(pvr *velerov1api.PodVolumeRestore) bool { + if tc.noChange { + return false + } + + pvr.Spec.Cancel = true + + return true + } + err = UpdatePVRWithRetry(ctx, r.client, namespacedName, velerotest.NewLogger().WithField("name", tc.Name), updateFunc) + if tc.ExpectErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/pkg/exposer/mocks/PodVolumeExposer.go b/pkg/exposer/mocks/PodVolumeExposer.go new file mode 100644 index 000000000..fbd5749ba --- /dev/null +++ b/pkg/exposer/mocks/PodVolumeExposer.go @@ -0,0 +1,125 @@ +// Code generated by mockery v2.39.1. DO NOT EDIT. + +package mocks + +import ( + context "context" + + client "sigs.k8s.io/controller-runtime/pkg/client" + + exposer "github.com/vmware-tanzu/velero/pkg/exposer" + + mock "github.com/stretchr/testify/mock" + + time "time" + + corev1api "k8s.io/api/core/v1" +) + +// PodVolumeExposer is an autogenerated mock type for the PodVolumeExposer type +type PodVolumeExposer struct { + mock.Mock +} + +// CleanUp provides a mock function with given fields: _a0, _a1 +func (_m *PodVolumeExposer) CleanUp(_a0 context.Context, _a1 corev1api.ObjectReference) { + _m.Called(_a0, _a1) +} + +// DiagnoseExpose provides a mock function with given fields: _a0, _a1 +func (_m *PodVolumeExposer) DiagnoseExpose(_a0 context.Context, _a1 corev1api.ObjectReference) string { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for DiagnoseExpose") + } + + var r0 string + if rf, ok := ret.Get(0).(func(context.Context, corev1api.ObjectReference) string); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + +// Expose provides a mock function with given fields: _a0, _a1, _a2 +func (_m *PodVolumeExposer) Expose(_a0 context.Context, _a1 corev1api.ObjectReference, _a2 exposer.PodVolumeExposeParam) error { + ret := _m.Called(_a0, _a1, _a2) + + if len(ret) == 0 { + panic("no return value specified for Expose") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, corev1api.ObjectReference, exposer.PodVolumeExposeParam) error); ok { + r0 = rf(_a0, _a1, _a2) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// GetExposed provides a mock function with given fields: _a0, _a1, _a2, _a3, _a4 +func (_m *PodVolumeExposer) GetExposed(_a0 context.Context, _a1 corev1api.ObjectReference, _a2 client.Client, _a3 string, _a4 time.Duration) (*exposer.ExposeResult, error) { + ret := _m.Called(_a0, _a1, _a2, _a3, _a4) + + if len(ret) == 0 { + panic("no return value specified for GetExposed") + } + + var r0 *exposer.ExposeResult + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, corev1api.ObjectReference, client.Client, string, time.Duration) (*exposer.ExposeResult, error)); ok { + return rf(_a0, _a1, _a2, _a3, _a4) + } + if rf, ok := ret.Get(0).(func(context.Context, corev1api.ObjectReference, client.Client, string, time.Duration) *exposer.ExposeResult); ok { + r0 = rf(_a0, _a1, _a2, _a3, _a4) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*exposer.ExposeResult) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, corev1api.ObjectReference, client.Client, string, time.Duration) error); ok { + r1 = rf(_a0, _a1, _a2, _a3, _a4) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// PeekExposed provides a mock function with given fields: _a0, _a1 +func (_m *PodVolumeExposer) PeekExposed(_a0 context.Context, _a1 corev1api.ObjectReference) error { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for PeekExposed") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, corev1api.ObjectReference) error); ok { + r0 = rf(_a0, _a1) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// NewPodVolumeExposer creates a new instance of PodVolumeExposer. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewPodVolumeExposer(t interface { + mock.TestingT + Cleanup(func()) +}) *PodVolumeExposer { + mock := &PodVolumeExposer{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/pkg/podvolume/restorer.go b/pkg/podvolume/restorer.go index 1c50b79d9..bebbeee3e 100644 --- a/pkg/podvolume/restorer.go +++ b/pkg/podvolume/restorer.go @@ -98,7 +98,7 @@ func newRestorer( return } - if pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseCompleted || pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseFailed { + if pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseCompleted || pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseFailed || pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseCanceled { r.resultsLock.Lock() defer r.resultsLock.Unlock() @@ -234,7 +234,7 @@ ForEachVolume: errs = append(errs, errors.New("timed out waiting for all PodVolumeRestores to complete")) break ForEachVolume case res := <-resultsChan: - if res.Status.Phase == velerov1api.PodVolumeRestorePhaseFailed { + if res.Status.Phase == velerov1api.PodVolumeRestorePhaseFailed || res.Status.Phase == velerov1api.PodVolumeRestorePhaseCanceled { errs = append(errs, errors.Errorf("pod volume restore failed: %s", res.Status.Message)) } tracker.TrackPodVolume(res)