...
As CollectionAlias works as an extra pointer to the existing collection in the RootCoordinator, we can implement collection hot reloading at a much lower cost compared to the 1, 2 approaches.
With CollectionAlias, we can implement collection hot reloading functionality as such:
- Create Collection
A& insert data. - Create CollectionAlias
Z->A. - Create Collection
B& insert data. - Update CollectionAlias
Z->B. - Users can access new data with the
Zalias.
Public Interfaces
New Public APIs
...
| Code Block |
|---|
// milvus.proto
message CreateAliasRequest {
common.MsgBase base = 1;
string collection_name = 2;
string alias = 3;
}
message DropAliasRequest {
common.MsgBase base = 1;
string collection_namealias = 2;
string alias = 3;
}
message AlterAliasRequest{
common.MsgBase base = 1;
string collection_name = 2;
string alias = 3;
}
service MilvusService {
// NEW
rpc CreateAlias(SetAliasRequest) returns (common.Status) {}
// NEW
rpc DropAlias(DropAliasRequest) returns (common.Status) {}
// NEW
rpc AlterAlias(AlterAliasRequest) returns (common.Status) {}
}
|
...
| Code Block |
|---|
type metaTable struct {
client kv.SnapShotKV // client of a reliable kv service, i.e. etcd client
tenantID2Meta map[typeutil.UniqueID]pb.TenantMeta // tenant id to tenant meta
proxyID2Meta map[typeutil.UniqueID]pb.ProxyMeta // proxy id to proxy meta
collID2Meta map[typeutil.UniqueID]pb.CollectionInfo // collection_id -> meta
collName2ID map[string]typeutil.UniqueID // collection name to collection id
// NEW
**collAlias2ID map[string]typeutil.UniqueID**
...
} |
As CollectionAlias & CollectionName are equal, GetCollectionByName also checks metaTable.collAlias2ID when getting the collection by name.
...
| Code Block |
|---|
// root_coord.proto
service RootCoord {
// NEW
// CreateAlias creates 1 to 1 mapping between `alias` and `collection_name`
// 1. If there no `alias` in the metaTable:
// 1.1 new `alias` will be added to the metaTable
// 1.2 `alias` will be persisted in the `etcd`
// 1.3 `dd_op` will sent to log broker.
// 2. If there is `alias/collection` in the metaTable:
// 2.1 An `alias/collection already exists` error will be returned.
rpc CreateAlias(milvus.CreateAliasRequest) returns (common.Status) {}
// NEW
// 1. DropAlias
// 1.1 Removes theexisting mapping from the metaTable
// 1.2 Removes theexisting mapping from the `etcd`
// 1.3 `dd_op` will be sent to the log broker.
// 1.4 Invalidates proxy caches
rpc DropAlias(milvus.DropAliasRequest) returns (common.Status) {}
// NEW
// 1. AlterAlias
// 1.1 Existing Mappingmapping will be updated in metaTable.
// 1.2 MappingExisting mapping will be updated in `etcd`
// 1.3 `dd_op` will be sent to log broker.
// 1.4 Invalidates proxy caches.
rpc AlterAlias(milvus.AlterAliasRequest) returns (common.Status) {}
// UPDATED REQUIRED
// Collection can't be dropped if it is referenced by an `alias`.
rpc DropCollection(DropCollectionRequest) returns (common.Status) {}
// UPDATED REQUIRED
// DescribeCollection now returns `aliases`
rpc DescribeCollection(DescribeCollectionRequest) returns (DescribeCollectionResponse) {}
}
|
There will be new tasks as `CreateAliasTask`, `DropAliasTask`, and `AlterAliasTask` that handles the actual implementation within the GRPC methods.
Recovery
The recovery process will handle CreateAliasDDType , DropAliasDDType, AlterAliasDDType types.
| Code Block |
|---|
func (c *Core) reSendDdMsg(ctx context.Context) error { flag, err := c.MetaTable.client.Load(DDMsgSendPrefix, 0) if err != nil || flag == "true" { log.Debug("No un-successful DdMsg") return nil } ddOpStr, err := c.MetaTable.client.Load(DDOperationPrefix, 0) if err != nil { log.Debug("DdOperation key does not exist") return nil } var ddOp DdOperation if err = json.Unmarshal([]byte(ddOpStr), &ddOp); err != nil { return err } switch ddOp.Type { case CreateCollectionDDType: var ddReq = internalpb.CreateCollectionRequest{} if err = proto.UnmarshalText(ddOp.Body, &ddReq); err != nil { return err } collInfo, err := c.MetaTable.GetCollectionByName(ddReq.CollectionName, 0) if err != nil { return err } if err = c.SendDdCreateCollectionReq(ctx, &ddReq, collInfo.PhysicalChannelNames); err != nil { return err } // NEW case CreateAliasDDType: ... // NEW case DropAliasDDType: ... // NEW case AlterAliasDDType: ... ... } |
Compatibility, Deprecation, and Migration Plan
...