 CurveFS对接S3方案设计S3Client() {} virtual ~S3Client() {} virtual void Init(curve::common::S3AdapterOption option) = 0; virtual int Upload(std::string name, const char* buf, uint64_t length) = 0; virtual int Append(std::string Append(std::string name, const char* buf, uint64_t length) = 0; virtual int Download(std::string name, char* buf, uint64_t offset, uint64_t length) = 0; }; metaserver.proto enum FileType { TYPE_DIRECTORY message UpdateInodeResponse {© XXX Page 7 of 11 required MetaStatusCode statusCode = 1; } service MetaServerService { rpc UpdateInodeS3Version(UpdateInodeS3VersionRequest) returns (UpdateIno0 码力 | 11 页 | 145.77 KB | 6 月前3 CurveFS对接S3方案设计S3Client() {} virtual ~S3Client() {} virtual void Init(curve::common::S3AdapterOption option) = 0; virtual int Upload(std::string name, const char* buf, uint64_t length) = 0; virtual int Append(std::string Append(std::string name, const char* buf, uint64_t length) = 0; virtual int Download(std::string name, char* buf, uint64_t offset, uint64_t length) = 0; }; metaserver.proto enum FileType { TYPE_DIRECTORY message UpdateInodeResponse {© XXX Page 7 of 11 required MetaStatusCode statusCode = 1; } service MetaServerService { rpc UpdateInodeS3Version(UpdateInodeS3VersionRequest) returns (UpdateIno0 码力 | 11 页 | 145.77 KB | 6 月前3
 Curve文件系统空间分配方案extents = 2; // } message DeallocateSpaceResponse { required StatusCode status = 1; } service MetaServerService {© XXX Page 9 of 11 rpc AllocateSpace(AllocateSpaceRequest) returns (AllocateSpaceResponse); 0; }; using Extents = std::vector Curve文件系统空间分配方案extents = 2; // } message DeallocateSpaceResponse { required StatusCode status = 1; } service MetaServerService {© XXX Page 9 of 11 rpc AllocateSpace(AllocateSpaceRequest) returns (AllocateSpaceResponse); 0; }; using Extents = std::vector- ; class Allocator { public: Allocator(...) {} virtual ~Allocator() = default;© XXX Page 10 of 11 /** * @brief * * @param size uint64_t */ virtual uint64_t Allocate(uint32_t size, const AllocateHint& allocateHint, Extents* extents) = 0; /** * @brief */ virtual void Deallocate(const 0 码力 | 11 页 | 159.17 KB | 6 月前3
 OID CND Asia Slide: CurveFSisolation Server Indicates a physical server Chunkserver is a service instance on a physical serverCurve metadata organization Curve maps virtual block devices to files Each file contains chunks scattered performance optimizations RAFT protocol Zero data copy pre-created file poolCURVE file system File service middleware Upper-layer applications can access file data in storage pools through interfaces such0 码力 | 24 页 | 3.47 MB | 6 月前3 OID CND Asia Slide: CurveFSisolation Server Indicates a physical server Chunkserver is a service instance on a physical serverCurve metadata organization Curve maps virtual block devices to files Each file contains chunks scattered performance optimizations RAFT protocol Zero data copy pre-created file poolCURVE file system File service middleware Upper-layer applications can access file data in storage pools through interfaces such0 码力 | 24 页 | 3.47 MB | 6 月前3
 CurveBS IO Processing Flowtopology structure of CURVE. CurveBS uses the central node Metadata Server (MDS) to manage virtual disk mapping and data replicas distribution. Decentralization currently leads to excessive complexity rollback from snapshot CurveBS data organization Data organization of virtual block device in CurveBS1. CurveBS maps the virtual block device to a file. For example, block device /dev/sda corresponds support 4KB aligned read/write on block devices.CurveBS file structure of virtual block device As mentioned above, CurveBS maps virtual block devices to files. Let's look at the metadata for a file in CurveBS0 码力 | 13 页 | 2.03 MB | 6 月前3 CurveBS IO Processing Flowtopology structure of CURVE. CurveBS uses the central node Metadata Server (MDS) to manage virtual disk mapping and data replicas distribution. Decentralization currently leads to excessive complexity rollback from snapshot CurveBS data organization Data organization of virtual block device in CurveBS1. CurveBS maps the virtual block device to a file. For example, block device /dev/sda corresponds support 4KB aligned read/write on block devices.CurveBS file structure of virtual block device As mentioned above, CurveBS maps virtual block devices to files. Let's look at the metadata for a file in CurveBS0 码力 | 13 页 | 2.03 MB | 6 月前3
 CurveFS S3本地缓存盘方案class DiskCacheManagerImpl : public DiskCacheManager{ public: DiskCacheManagerImpl(); virtual ~DiskCacheManagerImpl() { TrimStop(); } int Init(const std::string cacheDir); maxCachedObj; // ? bthread::Mutex mtx_; }; class DiskCacheWrite { public: DiskCacheWrite() {} virtual ~DiskCacheWrite() { AsyncUploadStop(); } int WriteDiskFile(std::string fullWriteDir std::string CacheDir_; }; class DiskCacheRead { public:© XXX Page 7 of 9 DiskCacheRead() {} virtual ~DiskCacheRead() {} int ReadDiskFile(std::string name, char* buf, uint64_t offset, uint64_t0 码力 | 9 页 | 150.46 KB | 6 月前3 CurveFS S3本地缓存盘方案class DiskCacheManagerImpl : public DiskCacheManager{ public: DiskCacheManagerImpl(); virtual ~DiskCacheManagerImpl() { TrimStop(); } int Init(const std::string cacheDir); maxCachedObj; // ? bthread::Mutex mtx_; }; class DiskCacheWrite { public: DiskCacheWrite() {} virtual ~DiskCacheWrite() { AsyncUploadStop(); } int WriteDiskFile(std::string fullWriteDir std::string CacheDir_; }; class DiskCacheRead { public:© XXX Page 7 of 9 DiskCacheRead() {} virtual ~DiskCacheRead() {} int ReadDiskFile(std::string name, char* buf, uint64_t offset, uint64_t0 码力 | 9 页 | 150.46 KB | 6 月前3
 Curve支持S3 数据缓存方案chunkPos_(chunkPos), len_(len) { data_ = new char[len]; memcpy(data_, data, len); } virtual ~DataCache() { delete data_; data_ = NULL; } void Write(uint32_t cachePos0 码力 | 9 页 | 179.72 KB | 6 月前3 Curve支持S3 数据缓存方案chunkPos_(chunkPos), len_(len) { data_ = new char[len]; memcpy(data_, data, len); } virtual ~DataCache() { delete data_; data_ = NULL; } void Write(uint32_t cachePos0 码力 | 9 页 | 179.72 KB | 6 月前3
 Curve for CNCF MainCurve can backup / sync data to slave cloud • When master cloud failure happens, Database service can move to the slave cloudData apps(middleware/bigdata/ai) • CurveFS can manage different storages native supportCloud native for CurveBS • CSI plugin for CurveBS • Deploy CurveBS as container service (in Plan) • Config CurveBS by (Cluster and Pool CRDs) in Kubernetes (in Plan) • Support Operator supportCloud native plan for CurveFS • CSI plugin for CurveFS (in Plan) • Deploy CurveFS as container service (in Plan) • Config CurveFS by (cluster and storage pools) CRDs in Kubernetes (in Plan) • Support0 码力 | 21 页 | 4.56 MB | 6 月前3 Curve for CNCF MainCurve can backup / sync data to slave cloud • When master cloud failure happens, Database service can move to the slave cloudData apps(middleware/bigdata/ai) • CurveFS can manage different storages native supportCloud native for CurveBS • CSI plugin for CurveBS • Deploy CurveBS as container service (in Plan) • Config CurveBS by (Cluster and Pool CRDs) in Kubernetes (in Plan) • Support Operator supportCloud native plan for CurveFS • CSI plugin for CurveFS (in Plan) • Deploy CurveFS as container service (in Plan) • Config CurveFS by (cluster and storage pools) CRDs in Kubernetes (in Plan) • Support0 码力 | 21 页 | 4.56 MB | 6 月前3
 Curve文件系统元数据Proto(接口定义)string fsName = 2; } message DeleteFsInfoResponse { required MetaStatusCode statusCode = 1; } service MdsService { // fs interface rpc CreateFs(CreateFsRequest) returns (CreateFsResponse); uint64 inodeId = 2; } message DeleteInodeResponse { required MetaStatusCode statusCode = 1; } service MetaServerService { // dentry interface rpc GetDentry(GetDentryRequest) returns (GetDentryResponse); required uint32 fsId = 1; } message UnInitSpaceResponse { required SpaceStatusCode status = 1; } service SpaceAllocService { // space interface rpc InitSpace(InitSpaceRequest) returns (InitSpaceResponse);0 码力 | 15 页 | 80.33 KB | 6 月前3 Curve文件系统元数据Proto(接口定义)string fsName = 2; } message DeleteFsInfoResponse { required MetaStatusCode statusCode = 1; } service MdsService { // fs interface rpc CreateFs(CreateFsRequest) returns (CreateFsResponse); uint64 inodeId = 2; } message DeleteInodeResponse { required MetaStatusCode statusCode = 1; } service MetaServerService { // dentry interface rpc GetDentry(GetDentryRequest) returns (GetDentryResponse); required uint32 fsId = 1; } message UnInitSpaceResponse { required SpaceStatusCode status = 1; } service SpaceAllocService { // space interface rpc InitSpace(InitSpaceRequest) returns (InitSpaceResponse);0 码力 | 15 页 | 80.33 KB | 6 月前3
 CurveFS ChunkID持久化AllocateS3ChunkResponse 的状态码的类型为 FSStatusCode; proto/space.proto/service SpaceAllocService中的 rpc AllocateS3Chunk 复制到 proto/space.proto/service/MdsService中; curvefs/src/mds/mds_services.h MdsServiceImp类中增加0 码力 | 3 页 | 79.38 KB | 6 月前3 CurveFS ChunkID持久化AllocateS3ChunkResponse 的状态码的类型为 FSStatusCode; proto/space.proto/service SpaceAllocService中的 rpc AllocateS3Chunk 复制到 proto/space.proto/service/MdsService中; curvefs/src/mds/mds_services.h MdsServiceImp类中增加0 码力 | 3 页 | 79.38 KB | 6 月前3
 Curve核心组件之snapshotclonedata object data object S3 Snap Task etcd mds client 3.获取快照元数据 datastore metastore http service clone Task user 快照元数据 2.创建内部快照 5.删除内部快照 快照数据 1.发起快照 SnapshotCloneServer 6.删除内部快照数据快照的元数据和数据组织 data object S3 Snap Task etcd MDS client 2.创建克隆卷 3.分配卷空间 7.拷贝数据 datastore metastore http service clone Task user 快照、克隆元数据 SnapshotCloneServer 1.发起克隆 5.8.9.更新元数据克隆卷状态变化 • 初始状态; • 正在安装元数据或拷贝数据中;0 码力 | 23 页 | 1.32 MB | 6 月前3 Curve核心组件之snapshotclonedata object data object S3 Snap Task etcd mds client 3.获取快照元数据 datastore metastore http service clone Task user 快照元数据 2.创建内部快照 5.删除内部快照 快照数据 1.发起快照 SnapshotCloneServer 6.删除内部快照数据快照的元数据和数据组织 data object S3 Snap Task etcd MDS client 2.创建克隆卷 3.分配卷空间 7.拷贝数据 datastore metastore http service clone Task user 快照、克隆元数据 SnapshotCloneServer 1.发起克隆 5.8.9.更新元数据克隆卷状态变化 • 初始状态; • 正在安装元数据或拷贝数据中;0 码力 | 23 页 | 1.32 MB | 6 月前3
共 12 条
- 1
- 2













