Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

issue-2674: directory sharding fixes + basic integration test #2796

Merged
merged 2 commits into from
Jan 7, 2025
Merged

issue-2674: directory sharding fixes + basic integration test #2796

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
59b8882
issue-2674: directory sharding fixes + basic integration test
qkrorlqr Jan 3, 2025
4130e94
issue-2674: directory sharding - ListNodes and GetNodeAttr ut
qkrorlqr Jan 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions cloud/filestore/apps/client/lib/command.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -494,15 +494,17 @@ TVector<TFileStoreCommand::TPathEntry> TFileStoreCommand::ResolvePath(
NProto::TListNodesResponse TFileStoreCommand::ListAll(
ISession& session,
const TString& fsId,
ui64 parentId)
ui64 parentId,
bool disableMultiTabletForwarding)
{
NProto::TListNodesResponse fullResult;
TString cookie;
do {
auto request = CreateRequest<NProto::TListNodesRequest>();
request->SetFileSystemId(fsId);
request->SetNodeId(parentId);
request->MutableHeaders()->SetDisableMultiTabletForwarding(true);
request->MutableHeaders()->SetDisableMultiTabletForwarding(
disableMultiTabletForwarding);
request->SetCookie(cookie);

auto response = WaitFor(session.ListNodes(
Expand Down
3 changes: 2 additions & 1 deletion cloud/filestore/apps/client/lib/command.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,8 @@ class TFileStoreCommand
NProto::TListNodesResponse ListAll(
ISession& session,
const TString& fsId,
ui64 parentId);
ui64 parentId,
bool disableMultiTabletForwarding);

class TSessionGuard final
{
Expand Down
2 changes: 1 addition & 1 deletion cloud/filestore/apps/client/lib/diff.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ class TDiffCommand final
ui64 parentId,
TMap<TString, NProto::TNodeAttr>& result)
{
auto response = ListAll(session, fsId, parentId);
auto response = ListAll(session, fsId, parentId, false);

for (ui32 i = 0; i < response.NodesSize(); ++i) {
const auto& node = response.GetNodes(i);
Expand Down
2 changes: 1 addition & 1 deletion cloud/filestore/apps/client/lib/find.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ class TFindCommand final
ui32 depth)
{
--depth;
auto response = ListAll(session, fsId, parentId);
auto response = ListAll(session, fsId, parentId, false);

// TODO: async

Expand Down
2 changes: 1 addition & 1 deletion cloud/filestore/apps/client/lib/find_garbage.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ class TFindGarbageCommand final
TVector<TNode>* nodes)
{
// TODO: async listing
auto response = ListAll(session, fsId, parentId);
auto response = ListAll(session, fsId, parentId, true);

for (ui32 i = 0; i < response.NodesSize(); ++i) {
const auto& node = response.GetNodes(i);
Expand Down
5 changes: 5 additions & 0 deletions cloud/filestore/libs/storage/service/service_actor_createfs.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,11 @@ void TCreateFileStoreActor::ConfigureShards(const TActorContext& ctx)
request->Record.SetFileSystemId(
FileStoreConfig.ShardConfigs[i].GetFileSystemId());
request->Record.SetShardNo(i + 1);
if (StorageConfig->GetDirectoryCreationInShardsEnabled()) {
for (const auto& shard: FileStoreConfig.ShardConfigs) {
request->Record.AddShardFileSystemIds(shard.GetFileSystemId());
}
}

LOG_INFO(
ctx,
Expand Down
14 changes: 14 additions & 0 deletions cloud/filestore/libs/storage/service/service_actor_createhandle.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,7 @@ void TCreateHandleActor::CreateHandleInShard(const TActorContext& ctx)
request->Record.SetNodeId(RootNodeId);
request->Record.SetName(LeaderResponse.GetShardNodeName());
request->Record.ClearShardFileSystemId();
request->Record.MutableHeaders()->SetBehaveAsDirectoryTablet(false);
// E_EXCLUSIVE flag should be unset in order not to get EEXIST from the
// shard
const auto exclusiveFlag =
Expand Down Expand Up @@ -168,6 +169,19 @@ void TCreateHandleActor::HandleCreateHandleResponse(

LeaderResponded = true;
LeaderResponse = std::move(msg->Record);
if (LeaderResponse.GetHandle()) {
LOG_DEBUG(
ctx,
TFileStoreComponents::SERVICE,
"CreateHandle - child node is managed by leader, node: %lu"
", handle: %lu",
LeaderResponse.GetNodeAttr().GetId(),
LeaderResponse.GetHandle());

ReplyAndDie(ctx, std::move(LeaderResponse));
return;
}

CreateHandleInShard(ctx);
}

Expand Down
25 changes: 25 additions & 0 deletions cloud/filestore/libs/storage/service/service_actor_getnodeattr.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,31 @@ void TStorageServiceActor::HandleGetNodeAttr(
return NCloud::Reply(ctx, *ev, std::move(response));
}

const NProto::TFileStore& filestore = session->FileStore;

auto& headers = *msg->Record.MutableHeaders();
headers.SetBehaveAsDirectoryTablet(
StorageConfig->GetDirectoryCreationInShardsEnabled());
if (auto shardNo = ExtractShardNo(msg->Record.GetNodeId())) {
// parent directory is managed by a shard
auto [shardId, error] = SelectShard(
ctx,
sessionId,
seqNo,
headers.GetDisableMultiTabletForwarding(),
TEvService::TGetNodeAttrMethod::Name,
msg->CallContext->RequestId,
filestore,
shardNo);
if (HasError(error)) {
auto response =
std::make_unique<TEvService::TEvGetNodeAttrResponse>(
std::move(error));
return NCloud::Reply(ctx, *ev, std::move(response));
}
msg->Record.SetFileSystemId(shardId);
}

auto [cookie, inflight] = CreateInFlightRequest(
TRequestInfo(ev->Sender, ev->Cookie, msg->CallContext),
session->MediaKind,
Expand Down
25 changes: 25 additions & 0 deletions cloud/filestore/libs/storage/service/service_actor_listnodes.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -641,6 +641,31 @@ void TStorageServiceActor::HandleListNodes(
return NCloud::Reply(ctx, *ev, std::move(response));
}

const NProto::TFileStore& filestore = session->FileStore;

auto& headers = *msg->Record.MutableHeaders();
headers.SetBehaveAsDirectoryTablet(
StorageConfig->GetDirectoryCreationInShardsEnabled());
if (auto shardNo = ExtractShardNo(msg->Record.GetNodeId())) {
// parent directory is managed by a shard
auto [shardId, error] = SelectShard(
ctx,
sessionId,
seqNo,
headers.GetDisableMultiTabletForwarding(),
TEvService::TListNodesMethod::Name,
msg->CallContext->RequestId,
filestore,
shardNo);
if (HasError(error)) {
auto response =
std::make_unique<TEvService::TEvListNodesResponse>(
std::move(error));
return NCloud::Reply(ctx, *ev, std::move(response));
}
msg->Record.SetFileSystemId(shardId);
}

auto [cookie, inflight] = CreateInFlightRequest(
TRequestInfo(ev->Sender, ev->Cookie, msg->CallContext),
session->MediaKind,
Expand Down
129 changes: 129 additions & 0 deletions cloud/filestore/libs/storage/service/service_ut_sharding.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4663,6 +4663,135 @@ Y_UNIT_TEST_SUITE(TStorageServiceShardingTest)
service.DestroyHandle(headers, fsId, nodeId2, handle2);
service.DestroyHandle(headers, fsId, nodeId3, handle3);
}

SERVICE_TEST_SID_SELECT_IN_LEADER_ONLY(
ShouldListNodesAndGetNodeAttrInDirectoryInShard)
{
config.SetMultiTabletForwardingEnabled(true);
config.SetDirectoryCreationInShardsEnabled(true);
TTestEnv env({}, config);
env.CreateSubDomain("nfs");

ui32 nodeIdx = env.CreateNode("nfs");

const TString fsId = "test";
const auto shard1Id = fsId + "-f1";
const auto shard2Id = fsId + "-f2";

TServiceClient service(env.GetRuntime(), nodeIdx);
service.CreateFileStore(fsId, 1'000);
service.CreateFileStore(shard1Id, 1'000);
service.CreateFileStore(shard2Id, 1'000);

ConfigureShards(service, fsId, shard1Id, shard2Id);

auto headers = service.InitSession(fsId, "client");

auto createNodeResponse = service.CreateNode(
headers,
TCreateNodeArgs::Directory(RootNodeId, "dir1"))->Record;
const auto dir1Id = createNodeResponse.GetNode().GetId();
UNIT_ASSERT_VALUES_EQUAL(1, ExtractShardNo(dir1Id));

service.CreateNode(
headers,
TCreateNodeArgs::File(dir1Id, "file1"));
service.CreateNode(
headers,
TCreateNodeArgs::File(dir1Id, "file2"));
service.CreateNode(
headers,
TCreateNodeArgs::File(dir1Id, "file3"));
service.CreateNode(
headers,
TCreateNodeArgs::File(dir1Id, "file4"));

auto listNodesResponse = service.ListNodes(
headers,
fsId,
dir1Id)->Record;

UNIT_ASSERT_VALUES_EQUAL(4, listNodesResponse.NamesSize());
UNIT_ASSERT_VALUES_EQUAL("file1", listNodesResponse.GetNames(0));
UNIT_ASSERT_VALUES_EQUAL("file2", listNodesResponse.GetNames(1));
UNIT_ASSERT_VALUES_EQUAL("file3", listNodesResponse.GetNames(2));
UNIT_ASSERT_VALUES_EQUAL("file4", listNodesResponse.GetNames(3));
TVector<std::pair<ui64, TString>> nodes(4);
for (ui32 i = 0; i < 4; ++i) {
nodes[i] = {
listNodesResponse.GetNodes(i).GetId(),
listNodesResponse.GetNames(i)};
UNIT_ASSERT_VALUES_UNEQUAL(0, nodes[i].first);
}

UNIT_ASSERT_VALUES_EQUAL(2, ExtractShardNo(nodes[0].first));
UNIT_ASSERT_VALUES_EQUAL(1, ExtractShardNo(nodes[1].first));
UNIT_ASSERT_VALUES_EQUAL(2, ExtractShardNo(nodes[2].first));
UNIT_ASSERT_VALUES_EQUAL(1, ExtractShardNo(nodes[3].first));

auto getAttrResponse = service.GetNodeAttr(
headers,
fsId,
RootNodeId,
"dir1")->Record;

UNIT_ASSERT_VALUES_EQUAL(dir1Id, getAttrResponse.GetNode().GetId());
UNIT_ASSERT_VALUES_EQUAL(
static_cast<ui32>(NProto::E_DIRECTORY_NODE),
getAttrResponse.GetNode().GetType());

getAttrResponse = service.GetNodeAttr(
headers,
fsId,
dir1Id,
"file1")->Record;

UNIT_ASSERT_VALUES_EQUAL(
nodes[0].first,
getAttrResponse.GetNode().GetId());
UNIT_ASSERT_VALUES_EQUAL(
static_cast<ui32>(NProto::E_REGULAR_NODE),
getAttrResponse.GetNode().GetType());

getAttrResponse = service.GetNodeAttr(
headers,
fsId,
dir1Id,
"file2")->Record;

UNIT_ASSERT_VALUES_EQUAL(
nodes[1].first,
getAttrResponse.GetNode().GetId());
UNIT_ASSERT_VALUES_EQUAL(
static_cast<ui32>(NProto::E_REGULAR_NODE),
getAttrResponse.GetNode().GetType());

getAttrResponse = service.GetNodeAttr(
headers,
fsId,
dir1Id,
"file3")->Record;

UNIT_ASSERT_VALUES_EQUAL(
nodes[2].first,
getAttrResponse.GetNode().GetId());
UNIT_ASSERT_VALUES_EQUAL(
static_cast<ui32>(NProto::E_REGULAR_NODE),
getAttrResponse.GetNode().GetType());

getAttrResponse = service.GetNodeAttr(
headers,
fsId,
dir1Id,
"file4")->Record;

UNIT_ASSERT_VALUES_EQUAL(
nodes[3].first,
getAttrResponse.GetNode().GetId());
UNIT_ASSERT_VALUES_EQUAL(
static_cast<ui32>(NProto::E_REGULAR_NODE),
getAttrResponse.GetNode().GetType());
}
}

} // namespace NCloud::NFileStore::NStorage
1 change: 1 addition & 0 deletions cloud/filestore/libs/storage/tablet/tablet_actor_createhandle.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,7 @@ bool TIndexTabletActor::PrepareTx_CreateHandle(

auto shardId = args.RequestShardId;
if (!BehaveAsShard(args.Request.GetHeaders())
&& !GetFileSystem().GetShardFileSystemIds().empty()
&& Config->GetShardIdSelectionInLeaderEnabled())
{
args.Error = SelectShard(0 /*fileSize*/, &shardId);
Expand Down
1 change: 1 addition & 0 deletions cloud/filestore/libs/storage/tablet/tablet_actor_createnode.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -463,6 +463,7 @@ bool TIndexTabletActor::PrepareTx_CreateNode(

if (!BehaveAsShard(args.Request.GetHeaders())
&& Config->GetShardIdSelectionInLeaderEnabled()
&& !GetFileSystem().GetShardFileSystemIds().empty()
&& (args.Attrs.GetType() == NProto::E_REGULAR_NODE
|| Config->GetDirectoryCreationInShardsEnabled()))
{
Expand Down
11 changes: 10 additions & 1 deletion cloud/filestore/libs/storage/tablet/tablet_state_data.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1434,7 +1434,16 @@ TReadAheadCacheStats TIndexTabletState::CalculateReadAheadCacheStats() const

NProto::TError TIndexTabletState::SelectShard(ui64 fileSize, TString* shardId)
{
return Impl->ShardBalancer.SelectShard(fileSize, shardId);
auto e = Impl->ShardBalancer.SelectShard(fileSize, shardId);
if (HasError(e)) {
return e;
}

if (*shardId == GetFileSystemId()) {
shardId->clear();
}

return e;
}

void TIndexTabletState::UpdateShardStats(const TVector<TShardStats>& stats)
Expand Down
5 changes: 5 additions & 0 deletions cloud/filestore/tests/client_sharded_dir/canondata/result.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"test.test_nonsharded_vs_sharded_fs": {
"uri": "file://test.test_nonsharded_vs_sharded_fs/results.txt"
}
}
25 changes: 25 additions & 0 deletions ...lestore/tests/client_sharded_dir/canondata/test.test_nonsharded_vs_sharded_fs/results.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
[
{
"Type": 2,
"Links": 1,
"Name": "a0",
"Mode": 511,
"Id": 2
},
{
"Type": 2,
"Links": 1,
"Name": "a1",
"Mode": 511,
"Id": 17
}
][
{
"ShardFileSystemId": "fs1_s1",
"Name": "a0"
},
{
"ShardFileSystemId": "fs1_s2",
"Name": "a1"
}
]
1 change: 1 addition & 0 deletions cloud/filestore/tests/client_sharded_dir/cloud-filestore-tests-client_sharded_dir
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
/home/astr/.ya/build/symres/a7e618c7f78ab2ed77a865c40d747705/cloud-filestore-tests-client_sharded_dir
8 changes: 8 additions & 0 deletions cloud/filestore/tests/client_sharded_dir/nfs-storage.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
MultiTabletForwardingEnabled: true
LargeDeletionMarkersEnabled: true
MaxFileBlocks: 536870912
AutomaticShardCreationEnabled: true
ShardAllocationUnit: 1073741824
AutomaticallyCreatedShardSize: 1073741824
ShardIdSelectionInLeaderEnabled: true
DirectoryCreationInShardsEnabled: true
Loading
Loading