/*
 * Copyright 2019 Bloomberg Finance LP
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#ifndef INCLUDED_BUILDBOXCASD_LOCALCASSERVICE_H
#define INCLUDED_BUILDBOXCASD_LOCALCASSERVICE_H

#include <buildboxcasd_metricnames.h>
#include <buildboxcasd_proxymodefixture.h>
#include <buildboxcasd_server.h>

#include <buildboxcommon_fileutils.h>
#include <buildboxcommon_temporarydirectory.h>
#include <buildboxcommon_timeutils.h>

#include <buildboxcommonmetrics_countingmetricvalue.h>
#include <buildboxcommonmetrics_distributionmetricvalue.h>
#include <buildboxcommonmetrics_durationmetricvalue.h>
#include <buildboxcommonmetrics_testingutils.h>

#include <gmock/gmock.h>
#include <grpcpp/client_context.h>
#include <grpcpp/impl/codegen/call.h>
#include <grpcpp/server_context.h>
#include <grpcpp/support/sync_stream.h>
#include <gtest/gtest.h>

#include <fstream>
#include <iostream>
#include <memory.h>
#include <stdlib.h>
#include <unistd.h>

#if __APPLE__
#define st_mtim st_mtimespec
#define st_atim st_atimespec
#endif

using namespace buildboxcasd;
using namespace buildboxcommon;
using namespace buildboxcommon::buildboxcommonmetrics;

namespace {
const auto digestFunctionInitializer = []() {
    buildboxcommon::DigestGenerator::init();
    return 0;
}();
} // namespace

void generateAndStoreFiles(std::map<std::string, std::string> *file_contents,
                           std::map<std::string, Digest> *file_digests,
                           std::shared_ptr<LocalCas> storage)
{
    (*file_contents)["file1.sh"] = "file1Contents...";
    (*file_digests)["file1.sh"] =
        DigestGenerator::hash((*file_contents)["file1.sh"]);

    storage->writeBlob((*file_digests)["file1.sh"],
                       (*file_contents)["file1.sh"]);

    (*file_contents)["file2.c"] = "file2: [...data...]";
    (*file_digests)["file2.c"] =
        DigestGenerator::hash((*file_contents)["file2.c"]);

    storage->writeBlob((*file_digests)["file2.c"],
                       (*file_contents)["file2.c"]);
}

static Digest
createDirectoryTree(const std::map<std::string, Digest> &file_digests,
                    std::shared_ptr<LocalCas> storage)
{

    /*
     *  root/
     *  |-- file1.sh*
     *  |-- link -> subdir1/file2.c
     *  |-- subdir1/
     *           |-- file2.c
     */

    // Creating `root/`:
    Directory root_directory;

    // Adding a file to it:
    FileNode *f1 = root_directory.add_files();
    f1->set_name("file1.sh");
    f1->set_is_executable(true);
    f1->mutable_digest()->CopyFrom(file_digests.at("file1.sh"));

    // Adding symlink (`link` -> `subdir1/file2.c`)
    SymlinkNode *link = root_directory.add_symlinks();
    link->set_name("link");
    link->set_target("subdir1/file2.c");

    // Creating `subdir1/`:
    Directory subdirectory;

    // Adding `subdir1/file2.c`:
    FileNode *f2 = subdirectory.add_files();
    f2->set_name("file2.c");
    f2->mutable_digest()->CopyFrom(file_digests.at("file2.c"));

    google::protobuf::Timestamp gtime;
    google::protobuf::util::TimeUtil::FromString("2000-01-12T00:00:00.012300Z",
                                                 &gtime);
    auto properties2 = f2->mutable_node_properties();
    properties2->mutable_mtime()->CopyFrom(gtime);

    // Adding `subdir1/` under `dirA/`:
    DirectoryNode *d1 = root_directory.add_directories();
    d1->set_name("subdir1");

    const auto serialized_subdirectory = subdirectory.SerializeAsString();
    const auto subdirectory_digest =
        DigestGenerator::hash(serialized_subdirectory);
    d1->mutable_digest()->CopyFrom(subdirectory_digest);

    const auto serialized_root_directory = root_directory.SerializeAsString();
    const auto root_directory_digest =
        DigestGenerator::hash(serialized_root_directory);

    // Storing the two Directory protos...
    storage->writeBlob(root_directory_digest, serialized_root_directory);
    storage->writeBlob(subdirectory_digest, serialized_subdirectory);

    return root_directory_digest;
}

static bool fileContentMatches(const std::string &file_path,
                               const std::string &data)
{
    std::ifstream file(file_path, std::ios::binary);
    std::stringstream file_content;
    file_content << file.rdbuf();

    return data == file_content.str();
}

class RemoteServerFixture : public CasProxyModeFixture {
  protected:
    RemoteServerFixture() : CasProxyModeFixture() {}

    void SetUp() override
    {
        generateAndStoreFiles(&file_contents, &file_digests, local_storage);
        root_directory_digest =
            createDirectoryTree(file_digests, local_storage);
    }

    template <class T>
    void getProtoFromLocalStorage(const Digest &digest, T &t)
    {
        const auto type_blob = readBlobFromLocalStorage(digest);
        t.ParseFromString(type_blob);
    }

    Digest root_directory_digest;

    std::map<std::string, Digest> file_digests;
    std::map<std::string, std::string> file_contents;
};

/*
 * Write content to a random file, in directory temp_dir and return the
 * filename or the full path to the file.
 */
std::string write_file_to_existing_dir(const std::string temp_dir,
                                       const std::string &content,
                                       bool return_full_path = false)
{
    std::string file_path = temp_dir + "/tempXXXXXX";
    // Create a temp file
    int test_file = mkstemp(&file_path[0]);

    // get the new filename created from mkstemp
    const std::string file_name = file_path.substr(file_path.rfind("/") + 1);

    // write content to file
    if (write(test_file, content.c_str(), content.size()) !=
        static_cast<ssize_t>(content.size())) {
        throw "failed to write test file";
    }
    // set the mtime to something arbitrary
    google::protobuf::Timestamp gtime;
    google::protobuf::util::TimeUtil::FromString("2012-11-02T01:23:11.015670Z",
                                                 &gtime);
    buildboxcommon::FileUtils::setFileMtime(
        test_file, buildboxcommon::TimeUtils::parse_timestamp(gtime));
    close(test_file);

    return return_full_path ? file_path : file_name;
}

TEST_P(RemoteServerFixture, SimpleTreeCapture)
{
    buildboxcommon::buildboxcommonmetrics::clearAllMetricCollection();

    CaptureTreeRequest request;
    CaptureTreeResponse response;
    request.set_instance_name(instance_name);

    // create temp dir.
    buildboxcommon::TemporaryDirectory temp_dir1;

    // create temp file in directory
    std::string content_string = "I'm buildboxcasd!";
    std::string file_name = write_file_to_existing_dir(
        std::string(temp_dir1.name()), content_string);

    // add directory to request "path".
    request.add_path(std::string(temp_dir1.name()));
    request.set_bypass_local_cache(false);

    // send request
    localcas_stub->CaptureTree(&client_context, request, &response);

    // check results match local FS structure
    ASSERT_EQ(response.responses().size(), request.path().size());

    for (const auto &r : response.responses()) {
        ASSERT_EQ(r.status().code(), grpc::StatusCode::OK);
        ASSERT_EQ(r.path(), std::string(temp_dir1.name()));
        // root_directory_digest is not set
        ASSERT_EQ(r.root_directory_digest(), Digest());
        // Check d_local_storage for blob/contents.
        // Get digest from tree message.
        Tree t;
        getProtoFromLocalStorage(r.tree_digest(), t);
        auto files = t.root().files();
        ASSERT_EQ(files.size(), 1);
        auto file = t.root().files(0);
        ASSERT_EQ(file.name(), file_name);
        ASSERT_EQ(readBlobFromLocalStorage(file.digest()), content_string);
    }

    ASSERT_TRUE(allCollectedByNameWithValues<
                buildboxcommonmetrics::CountingMetricValue>({
        {MetricNames::COUNTER_NUM_BLOBS_WRITTEN_TO_REMOTE,
         buildboxcommonmetrics::CountingMetricValue(3)},
        {MetricNames::COUNTER_NUM_BLOBS_CAPTURED,
         buildboxcommonmetrics::CountingMetricValue(
             2)}, /* File and `Directory` message. */
    }));
}

TEST_P(RemoteServerFixture, SimpleTreeCaptureLocalOnly)
{
    buildboxcommon::buildboxcommonmetrics::clearAllMetricCollection();

    CaptureTreeRequest request;
    CaptureTreeResponse response;
    request.set_instance_name(instance_name);

    // create temp dir.
    buildboxcommon::TemporaryDirectory temp_dir1;

    // create temp file in directory
    std::string content_string = "I'm buildboxcasd!";
    std::string file_name = write_file_to_existing_dir(
        std::string(temp_dir1.name()), content_string);

    // add directory to request "path".
    request.add_path(std::string(temp_dir1.name()));
    request.set_bypass_local_cache(false);
    request.set_skip_upload(true);

    // send request
    localcas_stub->CaptureTree(&client_context, request, &response);

    // check results match local FS structure
    ASSERT_EQ(response.responses().size(), request.path().size());

    for (const auto &r : response.responses()) {
        ASSERT_EQ(r.status().code(), grpc::StatusCode::OK);
        ASSERT_EQ(r.path(), std::string(temp_dir1.name()));

        // root_directory_digest is not set
        ASSERT_EQ(r.root_directory_digest(), Digest());

        // Check d_local_storage for blob/contents.
        // Get digest from tree message.
        Tree t;
        getProtoFromLocalStorage(r.tree_digest(), t);
        auto files = t.root().files();
        ASSERT_EQ(files.size(), 1);
        auto file = t.root().files(0);
        ASSERT_EQ(file.name(), file_name);
        ASSERT_TRUE(local_storage->hasBlob(file.digest()));
        ASSERT_FALSE(remote_storage->hasBlob(file.digest()));
        ASSERT_EQ(readBlobFromLocalStorage(file.digest()), content_string);
    }

    ASSERT_TRUE(allCollectedByNameWithValues<
                buildboxcommonmetrics::CountingMetricValue>({
        {MetricNames::COUNTER_NUM_BLOBS_CAPTURED,
         buildboxcommonmetrics::CountingMetricValue(
             2)}, /* File and `Directory` message. */
    }));
}

TEST_P(RemoteServerFixture, SimpleTreeCaptureDirectoryOnly)
{
    buildboxcommon::buildboxcommonmetrics::clearAllMetricCollection();

    CaptureTreeRequest request;
    CaptureTreeResponse response;
    request.set_instance_name(instance_name);

    // create temp dir.
    buildboxcommon::TemporaryDirectory temp_dir1;

    // create temp file in directory
    std::string content_string = "I'm buildboxcasd!";
    std::string file_name = write_file_to_existing_dir(
        std::string(temp_dir1.name()), content_string);

    // add directory to request "path".
    request.add_path(std::string(temp_dir1.name()));
    request.set_bypass_local_cache(false);

    // only request directory message as output
    request.set_output_directory_format(
        Command_OutputDirectoryFormat_DIRECTORY_ONLY);

    // send request
    localcas_stub->CaptureTree(&client_context, request, &response);

    // check results match local FS structure
    ASSERT_EQ(response.responses().size(), request.path().size());

    for (const auto &r : response.responses()) {
        ASSERT_EQ(r.status().code(), grpc::StatusCode::OK);
        ASSERT_EQ(r.path(), std::string(temp_dir1.name()));
        // tree_digest is not set
        ASSERT_EQ(r.tree_digest(), Digest());
        // check directory message
        Directory d;
        getProtoFromLocalStorage(r.root_directory_digest(), d);
        auto files = d.files();
        ASSERT_EQ(files.size(), 1);
        auto file = d.files(0);
        ASSERT_EQ(file.name(), file_name);
        ASSERT_EQ(readBlobFromLocalStorage(file.digest()), content_string);
    }

    ASSERT_TRUE(allCollectedByNameWithValues<
                buildboxcommonmetrics::CountingMetricValue>({
        {MetricNames::COUNTER_NUM_BLOBS_WRITTEN_TO_REMOTE,
         buildboxcommonmetrics::CountingMetricValue(2)},
        {MetricNames::COUNTER_NUM_BLOBS_CAPTURED,
         buildboxcommonmetrics::CountingMetricValue(
             2)}, /* File and `Directory` message. */
    }));
}

TEST_P(RemoteServerFixture, SimpleTreeCaptureWithMoveHint)
{
    ASSERT_TRUE(local_storage->externalFileMovesAllowed());
    buildboxcommon::buildboxcommonmetrics::clearAllMetricCollection();

    CaptureTreeRequest request;
    CaptureTreeResponse response;
    request.set_instance_name(instance_name);
    request.set_move_files(true);

    // Create a temporary directory (in the same FS as the CAS storage).
    const auto temp_dir = local_storage->createTemporaryDirectory();

    // Create a temporary file in that directory
    const std::string content_string = "This file should be moved.";
    const std::string file_name =
        write_file_to_existing_dir(temp_dir.strname(), content_string);
    const std::string file_path = temp_dir.strname() + "/" + file_name;

    // add directory to request "path".
    request.add_path(temp_dir.strname());
    request.set_bypass_local_cache(false);

    // send request
    localcas_stub->CaptureTree(&client_context, request, &response);

    // check results match local FS structure
    ASSERT_EQ(response.responses().size(), request.path().size());

    for (const auto &r : response.responses()) {
        ASSERT_EQ(r.status().code(), grpc::StatusCode::OK);
        ASSERT_EQ(r.path(), temp_dir.strname());
        // Check d_local_storage for blob/contents.
        // Get digest from tree message.
        Tree t;
        getProtoFromLocalStorage(r.tree_digest(), t);
        auto files = t.root().files();
        ASSERT_EQ(files.size(), 1);
        auto file = t.root().files(0);
        ASSERT_EQ(file.name(), file_name);
        ASSERT_EQ(readBlobFromLocalStorage(file.digest()), content_string);
    }

    ASSERT_TRUE(allCollectedByNameWithValues<
                buildboxcommonmetrics::CountingMetricValue>({
        {MetricNames::COUNTER_NUM_BLOBS_WRITTEN_TO_REMOTE,
         buildboxcommonmetrics::CountingMetricValue(3)},
        {MetricNames::COUNTER_NUM_BLOBS_CAPTURED,
         buildboxcommonmetrics::CountingMetricValue(
             2)}, /* File and `Directory` message. */
    }));

    // Since the conditions are met (the file is in the same FS as the local
    // CAS root and the owner matches the user running this process, the file
    // should have been moved into storage:
    ASSERT_FALSE(buildboxcommon::FileUtils::isRegularFile(file_path.c_str()));
}

TEST_P(RemoteServerFixture, SimpleTreeCaptureMtime)
{
    CaptureTreeRequest request;
    CaptureTreeResponse response;
    request.set_instance_name(instance_name);

    // create temp dir.
    buildboxcommon::TemporaryDirectory temp_dir1;

    // create temp file in directory
    std::string content_string = "I'm buildboxcasd!";
    std::string file_name = write_file_to_existing_dir(
        std::string(temp_dir1.name()), content_string);

    // add directory to request "path".
    request.add_path(std::string(temp_dir1.name()));
    request.set_bypass_local_cache(false);
    request.add_node_properties("mtime");

    // send request
    localcas_stub->CaptureTree(&client_context, request, &response);

    // check results match local FS structure
    ASSERT_EQ(response.responses().size(), request.path().size());

    for (const auto &r : response.responses()) {
        ASSERT_EQ(r.status().code(), grpc::StatusCode::OK);
        ASSERT_EQ(r.path(), std::string(temp_dir1.name()));
        // Check d_local_storage for blob/contents.
        // Get digest from tree message.
        Tree t;
        getProtoFromLocalStorage(r.tree_digest(), t);
        auto files = t.root().files();
        ASSERT_EQ(files.size(), 1);
        auto file = t.root().files(0);
        ASSERT_EQ(file.name(), file_name);
        ASSERT_EQ(readBlobFromLocalStorage(file.digest()), content_string);
        auto properties = file.node_properties();
        if (file.name() == "file2.sh") {
            ASSERT_TRUE(properties.has_mtime());
            google::protobuf::Timestamp gtime;
            google::protobuf::util::TimeUtil::FromString(
                "2000-01-12T00:00:00.012300Z", &gtime);
            ASSERT_EQ(properties.mtime(), gtime);
        }
    }
}

TEST_P(RemoteServerFixture, SimpleTreeCaptureWithRoot)
{
    buildboxcommon::buildboxcommonmetrics::clearAllMetricCollection();

    CaptureTreeRequest request;
    CaptureTreeResponse response;
    request.set_instance_name(instance_name);

    // create temp dir.
    buildboxcommon::TemporaryDirectory root_dir;
    std::string dir_path = root_dir.strname() + "/directory";
    FileUtils::createDirectory(dir_path.c_str());

    // create temp file in directory
    std::string content_string = "I'm buildboxcasd!";
    std::string file_name =
        write_file_to_existing_dir(dir_path, content_string);

    // add directory to request "path".
    request.set_root(root_dir.strname());
    request.add_path(std::string("directory"));
    request.set_bypass_local_cache(false);

    // send request
    localcas_stub->CaptureTree(&client_context, request, &response);

    // check results match local FS structure
    ASSERT_EQ(response.responses().size(), request.path().size());

    for (const auto &r : response.responses()) {
        ASSERT_EQ(r.status().code(), grpc::StatusCode::OK);
        ASSERT_EQ(r.path(), "directory");
        // Check d_local_storage for blob/contents.
        // Get digest from tree message.
        Tree t;
        getProtoFromLocalStorage(r.tree_digest(), t);
        auto files = t.root().files();
        ASSERT_EQ(files.size(), 1);
        auto file = t.root().files(0);
        ASSERT_EQ(file.name(), file_name);
        ASSERT_EQ(readBlobFromLocalStorage(file.digest()), content_string);
    }

    ASSERT_TRUE(allCollectedByNameWithValues<
                buildboxcommonmetrics::CountingMetricValue>({
        {MetricNames::COUNTER_NUM_BLOBS_WRITTEN_TO_REMOTE,
         buildboxcommonmetrics::CountingMetricValue(3)},
        {MetricNames::COUNTER_NUM_BLOBS_CAPTURED,
         buildboxcommonmetrics::CountingMetricValue(
             2)}, /* File and `Directory` message. */
    }));
}

TEST_P(RemoteServerFixture, SandboxedTreeCapture)
{
    buildboxcommon::buildboxcommonmetrics::clearAllMetricCollection();

    grpc::ClientContext ns_client_context;
    GetInstanceNameForNamespaceRequest ns_request;
    GetInstanceNameForNamespaceResponse ns_response;
    CaptureTreeRequest request;
    CaptureTreeResponse response;

    // create temp dir.
    buildboxcommon::TemporaryDirectory root_dir;

    // set up sandboxed instance rooted in the temp dir
    ns_request.set_instance_name(instance_name);
    ns_request.set_root(root_dir.strname());
    auto ns_status = localcas_stub->GetInstanceNameForNamespace(
        &ns_client_context, ns_request, &ns_response);
    if (std::find(TEST_INSTANCE_NAMES.begin(), TEST_INSTANCE_NAMES.end(),
                  instance_name) != TEST_INSTANCE_NAMES.end()) {
        // Request is denied if there is a static instance configured as that
        // would allow clients to bypass the sandboxing restriction.
        ASSERT_EQ(ns_status.error_code(),
                  grpc::StatusCode::FAILED_PRECONDITION);
        return;
    }
    ASSERT_TRUE(ns_status.ok());
    const std::string sandboxed_instance_name = ns_response.instance_name();
    request.set_instance_name(sandboxed_instance_name);

    // Create directory and test file in that directory
    std::string dir_path = root_dir.strname() + "/directory";
    FileUtils::createDirectory(dir_path.c_str());
    std::string content_string = "I'm buildboxcasd!";
    std::string file_name =
        write_file_to_existing_dir(dir_path, content_string);

    // Add directory to request "path". It must be absolute but the root
    // refers to the temporary `root_dir`.
    request.add_path(std::string("/directory"));

    localcas_stub->CaptureTree(&client_context, request, &response);

    // check results match local FS structure
    ASSERT_EQ(response.responses().size(), request.path().size());

    for (const auto &r : response.responses()) {
        ASSERT_EQ(r.status().code(), grpc::StatusCode::OK);
        ASSERT_EQ(r.path(), "/directory");
        // Check d_local_storage for blob/contents.
        // Get digest from tree message.
        Tree t;
        getProtoFromLocalStorage(r.tree_digest(), t);
        auto files = t.root().files();
        ASSERT_EQ(files.size(), 1);
        auto file = t.root().files(0);
        ASSERT_EQ(file.name(), file_name);
        ASSERT_EQ(readBlobFromLocalStorage(file.digest()), content_string);
    }

    ASSERT_TRUE(allCollectedByNameWithValues<
                buildboxcommonmetrics::CountingMetricValue>({
        {MetricNames::COUNTER_NUM_BLOBS_WRITTEN_TO_REMOTE,
         buildboxcommonmetrics::CountingMetricValue(3)},
        {MetricNames::COUNTER_NUM_BLOBS_CAPTURED,
         buildboxcommonmetrics::CountingMetricValue(
             2)}, /* File and `Directory` message. */
    }));
}

TEST_P(RemoteServerFixture, NestedDirectoryTreeCapture)
{
    CaptureTreeRequest request;
    CaptureTreeResponse response;
    request.set_instance_name(instance_name);

    /* Directory Structure
     * /tempxxxx
     *     /temp1
     *          file1
     *      /temp2
     *          file2
     * /tempxxxx
     *     /temp3
     *          file3
     */

    const buildboxcommon::TemporaryDirectory temp_dir1;
    const std::string temp_dir2 = "temp2";
    const std::string temp_dir2_path =
        std::string(temp_dir1.name()) + "/" + temp_dir2;
    const buildboxcommon::TemporaryDirectory temp_dir3;

    // create directory inside root temporary directory
    FileUtils::createDirectory(temp_dir2_path.c_str());

    const std::vector<std::string> content_vec = {
        "I'm buildboxcasd in temp1!", "I'm buildboxcasd in temp3!"};

    const std::vector<std::string> path_vec = {std::string(temp_dir1.name()),
                                               std::string(temp_dir3.name())};

    const std::vector<std::string> file_vec = {
        write_file_to_existing_dir(path_vec[0], content_vec[0]),
        write_file_to_existing_dir(path_vec[1], content_vec[1])};

    // specify temp2 content separately
    const std::string content_in_file2 = "I'm buildboxcasd in temp2!";
    const std::string file_in_temp_dir2 =
        write_file_to_existing_dir(temp_dir2_path, content_in_file2);

    request.add_path(path_vec[0]);
    request.add_path(path_vec[1]);

    // send request
    localcas_stub->CaptureTree(&client_context, request, &response);

    ASSERT_EQ(response.responses().size(), request.path().size());

    // Check results match local FS structure
    for (int i = 0; i < (int)path_vec.size(); ++i) {
        auto r = response.responses(i);
        ASSERT_EQ(r.status().code(), grpc::StatusCode::OK);
        ASSERT_EQ(r.path(), path_vec[i]);
        Tree t;
        getProtoFromLocalStorage(r.tree_digest(), t);
        auto files = t.root().files();
        // check subdirectory of root
        if (t.root().directories().size() != 0) {
            auto dirnode = t.root().directories(0);
            ASSERT_EQ(dirnode.name(), temp_dir2);
            Directory d;
            getProtoFromLocalStorage(dirnode.digest(), d);
            ASSERT_EQ(files.size(), 1);
            auto file = d.files(0);
            ASSERT_EQ(file.name(), file_in_temp_dir2);
            ASSERT_EQ(readBlobFromLocalStorage(file.digest()),
                      content_in_file2);
        }
        ASSERT_EQ(files.size(), 1);
        auto file = t.root().files(0);
        ASSERT_EQ(file.name(), file_vec[i]);
        ASSERT_EQ(readBlobFromLocalStorage(file.digest()), content_vec[i]);
    }
}

TEST_P(RemoteServerFixture, SimpleFileCaptureMoveHint)
{
    ASSERT_TRUE(local_storage->externalFileMovesAllowed());

    buildboxcommon::buildboxcommonmetrics::clearAllMetricCollection();

    CaptureFilesRequest request;
    CaptureFilesResponse response;
    request.set_instance_name(instance_name);
    request.set_move_files(true);

    // Create a temporary directory (in the same FS as the CAS storage).
    const auto temp_dir = local_storage->createTemporaryDirectory();

    const int size = 10;
    const std::string content_string = "I'm buildboxcasd!";

    std::vector<std::string> content_vec;
    for (int i = 0; i < size; ++i) {
        auto content = content_string + std::to_string(i);
        auto file_path =
            write_file_to_existing_dir(temp_dir.strname(), content, true);
        content_vec.push_back(content);
        request.add_path(file_path);
    }

    localcas_stub->CaptureFiles(&client_context, request, &response);

    ASSERT_EQ(response.responses().size(), size);

    for (int i = 0; i < size; ++i) {
        auto r = response.responses(i);
        ASSERT_EQ(r.status().code(), grpc::StatusCode::OK);
        ASSERT_EQ(readBlobFromLocalStorage(r.digest()), content_vec[i]);
    }
    ASSERT_TRUE(collectedByName<DurationMetricValue>(
        buildboxcasd::MetricNames::TIMER_NAME_LOCAL_CAS_CAPTURE_FILES));

    ASSERT_TRUE(allCollectedByNameWithValues<
                buildboxcommonmetrics::CountingMetricValue>({
        {MetricNames::COUNTER_NUM_BLOBS_WRITTEN_TO_REMOTE,
         buildboxcommonmetrics::CountingMetricValue(size)},
        {MetricNames::COUNTER_NUM_BLOBS_CAPTURED,
         buildboxcommonmetrics::CountingMetricValue(size)},
    }));

    // All the files were moved since they are in the same FS as the CAS and
    // owned by the user running this process.
    ASSERT_TRUE(buildboxcommon::FileUtils::directoryIsEmpty(temp_dir.name()));
}

TEST_P(RemoteServerFixture, SimpleFileCapture)
{
    buildboxcommon::buildboxcommonmetrics::clearAllMetricCollection();

    CaptureFilesRequest request;
    CaptureFilesResponse response;
    request.set_instance_name(instance_name);

    const buildboxcommon::TemporaryDirectory temp_dir1;
    const int size = 10;
    std::string content_string = "I'm buildboxcasd!";
    std::vector<std::string> content_vec;

    for (int i = 0; i < size; ++i) {
        auto content = content_string + std::to_string(i);
        auto file_path = write_file_to_existing_dir(
            std::string(temp_dir1.name()), content, true);
        content_vec.push_back(content);
        request.add_path(file_path);
    }

    localcas_stub->CaptureFiles(&client_context, request, &response);

    ASSERT_EQ(response.responses().size(), size);

    for (int i = 0; i < size; ++i) {
        auto r = response.responses(i);
        ASSERT_EQ(r.status().code(), grpc::StatusCode::OK);
        ASSERT_EQ(readBlobFromLocalStorage(r.digest()), content_vec[i]);
    }
    ASSERT_TRUE(collectedByName<DurationMetricValue>(
        buildboxcasd::MetricNames::TIMER_NAME_LOCAL_CAS_CAPTURE_FILES));

    ASSERT_TRUE(allCollectedByNameWithValues<
                buildboxcommonmetrics::CountingMetricValue>({
        {MetricNames::COUNTER_NUM_BLOBS_WRITTEN_TO_REMOTE,
         buildboxcommonmetrics::CountingMetricValue(size)},
        {MetricNames::COUNTER_NUM_BLOBS_CAPTURED,
         buildboxcommonmetrics::CountingMetricValue(size)},
    }));
}

TEST_P(RemoteServerFixture, SimpleFileCaptureLocalOnly)
{
    buildboxcommon::buildboxcommonmetrics::clearAllMetricCollection();

    CaptureFilesRequest request;
    CaptureFilesResponse response;
    request.set_instance_name(instance_name);
    request.set_skip_upload(true);

    const buildboxcommon::TemporaryDirectory temp_dir1;
    const int size = 10;
    std::string content_string = "I'm buildboxcasd!";
    std::vector<std::string> content_vec;

    for (int i = 0; i < size; ++i) {
        auto content = content_string + std::to_string(i);
        auto file_path = write_file_to_existing_dir(
            std::string(temp_dir1.name()), content, true);
        content_vec.push_back(content);
        request.add_path(file_path);
    }

    localcas_stub->CaptureFiles(&client_context, request, &response);

    ASSERT_EQ(response.responses().size(), size);

    for (int i = 0; i < size; ++i) {
        auto r = response.responses(i);
        ASSERT_EQ(r.status().code(), grpc::StatusCode::OK);
        ASSERT_TRUE(local_storage->hasBlob(r.digest()));
        ASSERT_FALSE(remote_storage->hasBlob(r.digest()));
        ASSERT_EQ(readBlobFromLocalStorage(r.digest()), content_vec[i]);
    }
    ASSERT_TRUE(collectedByName<DurationMetricValue>(
        buildboxcasd::MetricNames::TIMER_NAME_LOCAL_CAS_CAPTURE_FILES));

    ASSERT_TRUE(allCollectedByNameWithValues<
                buildboxcommonmetrics::CountingMetricValue>({
        {MetricNames::COUNTER_NUM_BLOBS_CAPTURED,
         buildboxcommonmetrics::CountingMetricValue(size)},
    }));
}

TEST_P(RemoteServerFixture, SimpleFileCaptureWithMtime)
{
    CaptureFilesRequest request;
    CaptureFilesResponse response;
    request.set_instance_name(instance_name);
    const std::string property = "mtime";
    request.add_node_properties(property);

    const buildboxcommon::TemporaryDirectory temp_dir1;
    const int size = 10;
    std::string content_string = "I'm buildboxcasd!";
    std::vector<std::string> content_vec;

    for (int i = 0; i < size; ++i) {
        auto content = content_string + std::to_string(i);
        auto file_path = write_file_to_existing_dir(
            std::string(temp_dir1.name()), content, true);
        content_vec.push_back(content);
        request.add_path(file_path);
        request.add_node_properties(property);
    }

    localcas_stub->CaptureFiles(&client_context, request, &response);

    ASSERT_EQ(response.responses().size(), size);

    for (int i = 0; i < size; ++i) {
        auto r = response.responses(i);
        ASSERT_EQ(r.status().code(), grpc::StatusCode::OK);
        ASSERT_EQ(readBlobFromLocalStorage(r.digest()), content_vec[i]);
        ASSERT_EQ(r.is_executable(), 0);
        auto properties = r.node_properties();
        ASSERT_TRUE(properties.has_mtime());
        // FIXME: runner fs may not support sub-second precision
        auto timestamp = properties.mtime();
        google::protobuf::Timestamp exp_gtime1, exp_gtime2;
        google::protobuf::util::TimeUtil::FromString(
            "2012-11-02T01:23:11.015670Z", &exp_gtime1);
        google::protobuf::util::TimeUtil::FromString("2012-11-02T01:23:11Z",
                                                     &exp_gtime2);
        ASSERT_TRUE((timestamp == exp_gtime1) || (timestamp == exp_gtime2));
    }
}

TEST_P(RemoteServerFixture, SimpleFileCaptureWithRoot)
{
    buildboxcommon::buildboxcommonmetrics::clearAllMetricCollection();

    CaptureFilesRequest request;
    CaptureFilesResponse response;
    request.set_instance_name(instance_name);

    // Create a temporary directory (in the same FS as the CAS storage).
    const auto temp_dir = local_storage->createTemporaryDirectory();
    request.set_root(temp_dir.strname());

    const int size = 10;
    const std::string content_string = "I'm buildboxcasd!";

    std::vector<std::string> content_vec;
    for (int i = 0; i < size; ++i) {
        auto content = content_string + std::to_string(i);
        auto file_name =
            write_file_to_existing_dir(temp_dir.strname(), content, false);
        content_vec.push_back(content);
        request.add_path(file_name);
    }

    localcas_stub->CaptureFiles(&client_context, request, &response);

    ASSERT_EQ(response.responses().size(), size);

    for (int i = 0; i < size; ++i) {
        auto r = response.responses(i);
        ASSERT_EQ(r.status().code(), grpc::StatusCode::OK);
        ASSERT_EQ(readBlobFromLocalStorage(r.digest()), content_vec[i]);
    }
    ASSERT_TRUE(collectedByName<DurationMetricValue>(
        buildboxcasd::MetricNames::TIMER_NAME_LOCAL_CAS_CAPTURE_FILES));

    ASSERT_TRUE(allCollectedByNameWithValues<
                buildboxcommonmetrics::CountingMetricValue>({
        {MetricNames::COUNTER_NUM_BLOBS_WRITTEN_TO_REMOTE,
         buildboxcommonmetrics::CountingMetricValue(size)},
        {MetricNames::COUNTER_NUM_BLOBS_CAPTURED,
         buildboxcommonmetrics::CountingMetricValue(size)},
    }));
}

TEST_P(RemoteServerFixture, AbsoluteFileCaptureWithRoot)
{
    CaptureFilesRequest request;
    CaptureFilesResponse response;
    request.set_instance_name(instance_name);

    // Create a temporary directory (in the same FS as the CAS storage).
    const auto temp_dir = local_storage->createTemporaryDirectory();
    request.set_root(temp_dir.strname());
    request.add_path("/bin/sh");

    localcas_stub->CaptureFiles(&client_context, request, &response);

    ASSERT_EQ(response.responses().size(), 1);

    // While /bin/sh exists in the real filesystem root, it does not exist
    // in the specified root directory. This verifies that absolute paths
    // are resolved correctly when a root directory is specified.
    auto r = response.responses(0);
    ASSERT_EQ(r.status().code(), grpc::StatusCode::NOT_FOUND);
}

TEST_P(RemoteServerFixture, SimpleFileCaptureWithRootMoveHint)
{
    ASSERT_TRUE(local_storage->externalFileMovesAllowed());

    CaptureFilesRequest request;
    CaptureFilesResponse response;
    request.set_instance_name(instance_name);
    request.set_move_files(true);

    // Create a temporary directory (in the same FS as the CAS storage).
    const auto temp_dir = local_storage->createTemporaryDirectory();
    request.set_root(temp_dir.strname());

    const int size = 10;
    const std::string content_string = "I'm buildboxcasd!";

    std::vector<std::string> content_vec;
    for (int i = 0; i < size; ++i) {
        auto content = content_string + std::to_string(i);
        auto file_name =
            write_file_to_existing_dir(temp_dir.strname(), content, false);
        content_vec.push_back(content);
        request.add_path(file_name);
    }

    localcas_stub->CaptureFiles(&client_context, request, &response);

    ASSERT_EQ(response.responses().size(), size);

    for (int i = 0; i < size; ++i) {
        auto r = response.responses(i);
        ASSERT_EQ(r.status().code(), grpc::StatusCode::OK);
        ASSERT_EQ(readBlobFromLocalStorage(r.digest()), content_vec[i]);
    }

    // Verify that files were not moved despite the `move_files` hint, as
    // moving files is not supported in combination with safe path resolution
    // (which is required as `root` directory field was set).
    ASSERT_FALSE(buildboxcommon::FileUtils::directoryIsEmpty(temp_dir.name()));
}

TEST_P(RemoteServerFixture, SandboxedFileCapture)
{
    buildboxcommon::buildboxcommonmetrics::clearAllMetricCollection();

    grpc::ClientContext ns_client_context;
    GetInstanceNameForNamespaceRequest ns_request;
    GetInstanceNameForNamespaceResponse ns_response;
    CaptureFilesRequest request;
    CaptureFilesResponse response;

    // create temp dir.
    buildboxcommon::TemporaryDirectory root_dir;

    // set up sandboxed instance rooted in the temp dir
    ns_request.set_instance_name(instance_name);
    ns_request.set_root(root_dir.strname());
    auto ns_status = localcas_stub->GetInstanceNameForNamespace(
        &ns_client_context, ns_request, &ns_response);
    if (std::find(TEST_INSTANCE_NAMES.begin(), TEST_INSTANCE_NAMES.end(),
                  instance_name) != TEST_INSTANCE_NAMES.end()) {
        // Request is denied if there is a static instance configured as that
        // would allow clients to bypass the sandboxing restriction.
        ASSERT_EQ(ns_status.error_code(),
                  grpc::StatusCode::FAILED_PRECONDITION);
        return;
    }
    ASSERT_TRUE(ns_status.ok());
    const std::string sandboxed_instance_name = ns_response.instance_name();
    request.set_instance_name(sandboxed_instance_name);

    // Create test files
    const int size = 10;
    const std::string content_string = "I'm buildboxcasd!";
    std::vector<std::string> content_vec;
    for (int i = 0; i < size; ++i) {
        auto content = content_string + std::to_string(i);
        auto file_name =
            write_file_to_existing_dir(root_dir.strname(), content, false);
        content_vec.push_back(content);
        request.add_path("/" + file_name);
    }

    localcas_stub->CaptureFiles(&client_context, request, &response);

    ASSERT_EQ(response.responses().size(), size);

    for (int i = 0; i < size; ++i) {
        auto r = response.responses(i);
        ASSERT_EQ(r.status().code(), grpc::StatusCode::OK);
        ASSERT_EQ(readBlobFromLocalStorage(r.digest()), content_vec[i]);
    }
    ASSERT_TRUE(collectedByName<DurationMetricValue>(
        buildboxcasd::MetricNames::TIMER_NAME_LOCAL_CAS_CAPTURE_FILES));

    ASSERT_TRUE(allCollectedByNameWithValues<
                buildboxcommonmetrics::CountingMetricValue>({
        {MetricNames::COUNTER_NUM_BLOBS_WRITTEN_TO_REMOTE,
         buildboxcommonmetrics::CountingMetricValue(size)},
        {MetricNames::COUNTER_NUM_BLOBS_CAPTURED,
         buildboxcommonmetrics::CountingMetricValue(size)},
    }));
}

TEST_P(RemoteServerFixture, RemoteCasUpload)
{
    CaptureTreeRequest request;
    CaptureTreeResponse response;
    request.set_instance_name(instance_name);

    // send simple TreeCapture
    // create temp dir.
    buildboxcommon::TemporaryDirectory temp_dir1;
    // create temp file in directory
    std::string content_string = "I'm buildboxcasd!";
    std::string file_name = write_file_to_existing_dir(
        std::string(temp_dir1.name()), content_string);
    request.add_path(std::string(temp_dir1.name()));
    request.set_bypass_local_cache(false);

    // send request
    localcas_stub->CaptureTree(&client_context, request, &response);

    // check results match local FS structure
    ASSERT_EQ(response.responses().size(), request.path().size());

    auto resp = response.responses(0);
    ASSERT_EQ(resp.status().code(), grpc::StatusCode::OK);
    ASSERT_EQ(resp.path(), std::string(temp_dir1.name()));

    // fetch from remote CAS
    std::string tree_message = cas_client->fetchString(resp.tree_digest());

    // fetch from local CAS
    Tree t;
    this->getProtoFromLocalStorage(resp.tree_digest(), t);

    // check that it equals the one stored locally
    ASSERT_EQ(tree_message, t.SerializeAsString());
    ASSERT_TRUE(collectedByName<DurationMetricValue>(
        buildboxcasd::MetricNames::TIMER_NAME_LOCAL_CAS_CAPTURE_TREE));
}

TEST_P(RemoteServerFixture, StageRootDigestNotInCAS)
{
    StageTreeRequest request;
    StageTreeResponse response;
    request.set_instance_name(instance_name);

    Digest d(DigestGenerator::hash(""));
    d.set_hash("hash1234");
    request.mutable_root_digest()->CopyFrom(d);

    auto reader_writer = localcas_stub->StageTree(&client_context);
    ASSERT_TRUE(reader_writer->Write(request));

    ASSERT_FALSE(reader_writer->Read(&response));

    const auto status = reader_writer->Finish();
    ASSERT_EQ(status.error_code(), grpc::StatusCode::FAILED_PRECONDITION);

    ASSERT_FALSE(collectedByName<DurationMetricValue>(
        MetricNames::TIMER_NAME_LOCAL_CAS_STAGE_TREE_STAGE));
}

TEST_P(RemoteServerFixture, Stage)
{
    // Preparing a (valid) request:
    StageTreeRequest request;
    request.set_instance_name(instance_name);
    request.mutable_root_digest()->CopyFrom(root_directory_digest);
    ASSERT_TRUE(local_storage->hasBlob(root_directory_digest));

    // Stage():
    auto reader_writer = localcas_stub->StageTree(&client_context);
    ASSERT_TRUE(reader_writer->Write(request));

    // The tree was staged and we got a valid path in the response:
    StageTreeResponse response;
    ASSERT_TRUE(reader_writer->Read(&response));
    const std::string stage_path = response.path();
    ASSERT_TRUE(FileUtils::isDirectory(stage_path.c_str()));

    // The contents that were staged match the tree we requested:
    const std::string file1_stage_path = stage_path + "/file1.sh";
    ASSERT_TRUE(FileUtils::isRegularFile(file1_stage_path.c_str()));
    ASSERT_TRUE(FileUtils::isExecutable(file1_stage_path.c_str()));
    ASSERT_TRUE(
        fileContentMatches(file1_stage_path, file_contents["file1.sh"]));

    const std::string file2_stage_path = stage_path + "/subdir1/file2.c";
    ASSERT_TRUE(FileUtils::isRegularFile(file2_stage_path.c_str()));
    ASSERT_TRUE(
        fileContentMatches(file2_stage_path, file_contents["file2.c"]));
    // check the file mtime is as expected
    struct stat file2_stat;
    ASSERT_EQ(stat(file2_stage_path.c_str(), &file2_stat), 0);
    ASSERT_EQ(file2_stat.st_mtim.tv_sec, 947635200);
    // FIXME: runner fs may not support sub-second precision
    const long int nsec = file2_stat.st_mtim.tv_nsec;
    ASSERT_TRUE((nsec == 12300000) || (nsec == 0));

    const std::string link_stage_path = stage_path + "/link";
    ASSERT_TRUE(FileUtils::isRegularFile(link_stage_path.c_str()));
    ASSERT_TRUE(fileContentMatches(link_stage_path, file_contents["file2.c"]));

    // Ask the server to clean up:
    ASSERT_TRUE(reader_writer->Write(StageTreeRequest()));

    // Receive its last empty reply:
    ASSERT_TRUE(reader_writer->Read(&response));
    ASSERT_TRUE(response.path().empty());
    const auto status = reader_writer->Finish();
    ASSERT_EQ(status.error_code(), grpc::StatusCode::OK);

    // Make sure the staged directory was cleaned up
    ASSERT_FALSE(FileUtils::isDirectory(stage_path.c_str()));

    ASSERT_TRUE(allCollectedByName<DurationMetricValue>(
        {{MetricNames::TIMER_NAME_LOCAL_CAS_STAGE_TREE_TOTAL},
         {MetricNames::TIMER_NAME_LOCAL_CAS_STAGE_TREE_PREPARE},
         {MetricNames::TIMER_NAME_LOCAL_CAS_STAGE_TREE_STAGE}}));

    // Neither the root directory nor the subdirectory was in the tree cache
    ASSERT_TRUE(allCollectedByNameWithValues<CountingMetricValue>(
        {{MetricNames::COUNTER_NAME_LOCAL_CAS_TREE_CACHE_HITS,
          CountingMetricValue(0)},
         {MetricNames::COUNTER_NAME_LOCAL_CAS_TREE_CACHE_MISSES,
          CountingMetricValue(2)}}));

    // 0% tree cache hit rate
    ASSERT_TRUE(collectedByNameWithValue<DistributionMetricValue>(
        MetricNames::DISTRIBUTION_NAME_LOCAL_CAS_TREE_CACHE_HIT_PERCENTAGE,
        DistributionMetricValue(0)));
}

TEST_P(RemoteServerFixture, StageWithStagingMode)
{
    // Preparing a (valid) request:
    StageTreeRequest request;
    request.set_instance_name(instance_name);
    request.mutable_root_digest()->CopyFrom(root_directory_digest);
    // Explicit choose copy-stager
    request.set_staging_mode(StageTreeRequest_StagingMode_COPY_OR_LINK);
    ASSERT_TRUE(local_storage->hasBlob(root_directory_digest));

    // Stage():
    auto reader_writer = localcas_stub->StageTree(&client_context);
    ASSERT_TRUE(reader_writer->Write(request));

    // The tree was staged and we got a valid path in the response:
    StageTreeResponse response;
    ASSERT_TRUE(reader_writer->Read(&response));
    const std::string stage_path = response.path();
    ASSERT_TRUE(FileUtils::isDirectory(stage_path.c_str()));

    // The contents that were staged match the tree we requested:
    const std::string file1_stage_path = stage_path + "/file1.sh";
    ASSERT_TRUE(FileUtils::isRegularFile(file1_stage_path.c_str()));
    ASSERT_TRUE(FileUtils::isExecutable(file1_stage_path.c_str()));
    ASSERT_TRUE(
        fileContentMatches(file1_stage_path, file_contents["file1.sh"]));

    const std::string file2_stage_path = stage_path + "/subdir1/file2.c";
    ASSERT_TRUE(FileUtils::isRegularFile(file2_stage_path.c_str()));
    ASSERT_TRUE(
        fileContentMatches(file2_stage_path, file_contents["file2.c"]));
    // check the file mtime is as expected
    struct stat file2_stat;
    ASSERT_EQ(stat(file2_stage_path.c_str(), &file2_stat), 0);
    ASSERT_EQ(file2_stat.st_mtim.tv_sec, 947635200);
    // runner fs may not support sub-second precision
    const long int nsec = file2_stat.st_mtim.tv_nsec;
    ASSERT_TRUE((nsec == 12300000) || (nsec == 0));

    const std::string link_stage_path = stage_path + "/link";
    ASSERT_TRUE(FileUtils::isRegularFile(link_stage_path.c_str()));
    ASSERT_TRUE(fileContentMatches(link_stage_path, file_contents["file2.c"]));

    // Ask the server to clean up:
    ASSERT_TRUE(reader_writer->Write(StageTreeRequest()));

    // Receive its last empty reply:
    ASSERT_TRUE(reader_writer->Read(&response));
    ASSERT_TRUE(response.path().empty());
    const auto status = reader_writer->Finish();
    ASSERT_EQ(status.error_code(), grpc::StatusCode::OK);

    // Make sure the staged directory was cleaned up
    ASSERT_FALSE(FileUtils::isDirectory(stage_path.c_str()));

    ASSERT_TRUE(allCollectedByName<DurationMetricValue>(
        {{MetricNames::TIMER_NAME_LOCAL_CAS_STAGE_TREE_TOTAL},
         {MetricNames::TIMER_NAME_LOCAL_CAS_STAGE_TREE_PREPARE},
         {MetricNames::TIMER_NAME_LOCAL_CAS_STAGE_TREE_STAGE}}));

    // Neither the root directory nor the subdirectory was in the tree cache
    ASSERT_TRUE(allCollectedByNameWithValues<CountingMetricValue>(
        {{MetricNames::COUNTER_NAME_LOCAL_CAS_TREE_CACHE_HITS,
          CountingMetricValue(0)},
         {MetricNames::COUNTER_NAME_LOCAL_CAS_TREE_CACHE_MISSES,
          CountingMetricValue(2)}}));

    // 0% tree cache hit rate
    ASSERT_TRUE(collectedByNameWithValue<DistributionMetricValue>(
        MetricNames::DISTRIBUTION_NAME_LOCAL_CAS_TREE_CACHE_HIT_PERCENTAGE,
        DistributionMetricValue(0)));
}

TEST_P(RemoteServerFixture, StageSingleUser)
{
    // Preparing a (valid) request:
    StageTreeRequest request;
    request.set_instance_name(instance_name);
    request.mutable_root_digest()->CopyFrom(root_directory_digest);
    request.mutable_access_credentials()->set_uid(geteuid());
    ASSERT_TRUE(local_storage->hasBlob(root_directory_digest));

    // StageTree():
    auto reader_writer = localcas_stub->StageTree(&client_context);
    ASSERT_TRUE(reader_writer->Write(request));

    // The tree was staged and we got a valid path in the response:
    StageTreeResponse response;
    ASSERT_TRUE(reader_writer->Read(&response));
    const std::string stage_path = response.path();
    ASSERT_TRUE(FileUtils::isDirectory(stage_path.c_str()));

    // The contents that were staged match the tree we requested:
    const std::string file1_stage_path = stage_path + "/file1.sh";
    ASSERT_TRUE(FileUtils::isRegularFile(file1_stage_path.c_str()));
    ASSERT_TRUE(FileUtils::isExecutable(file1_stage_path.c_str()));
    ASSERT_TRUE(
        fileContentMatches(file1_stage_path, file_contents["file1.sh"]));
    struct stat file1_stat;
    ASSERT_EQ(stat(file1_stage_path.c_str(), &file1_stat), 0);

    // Verify that the file is not hardlinked to protect the cache from
    // corruption when running as a single user. FUSE, full file copies
    // and reflinks are safe options.
    EXPECT_EQ(file1_stat.st_nlink, 1);

    // Ask the server to clean up:
    ASSERT_TRUE(reader_writer->Write(StageTreeRequest()));

    // Receive its last empty reply:
    ASSERT_TRUE(reader_writer->Read(&response));
    ASSERT_TRUE(response.path().empty());
    const auto status = reader_writer->Finish();
    ASSERT_EQ(status.error_code(), grpc::StatusCode::OK);

    ASSERT_TRUE(allCollectedByName<DurationMetricValue>(
        {{MetricNames::TIMER_NAME_LOCAL_CAS_STAGE_TREE_TOTAL},
         {MetricNames::TIMER_NAME_LOCAL_CAS_STAGE_TREE_PREPARE},
         {MetricNames::TIMER_NAME_LOCAL_CAS_STAGE_TREE_STAGE}}));

    // Neither the root directory nor the subdirectory was in the tree cache
    ASSERT_TRUE(allCollectedByNameWithValues<CountingMetricValue>(
        {{MetricNames::COUNTER_NAME_LOCAL_CAS_TREE_CACHE_HITS,
          CountingMetricValue(0)},
         {MetricNames::COUNTER_NAME_LOCAL_CAS_TREE_CACHE_MISSES,
          CountingMetricValue(2)}}));

    // 0% tree cache hit rate
    ASSERT_TRUE(collectedByNameWithValue<DistributionMetricValue>(
        MetricNames::DISTRIBUTION_NAME_LOCAL_CAS_TREE_CACHE_HIT_PERCENTAGE,
        DistributionMetricValue(0)));
}

TEST_P(RemoteServerFixture, FetchStage)
{
    // First fetch the tree
    {
        grpc::ClientContext client_context;
        FetchTreeRequest request;
        FetchTreeResponse response;
        request.set_instance_name(instance_name);
        request.mutable_root_digest()->CopyFrom(root_directory_digest);
        request.set_fetch_file_blobs(true);
        const auto fetchStatus =
            localcas_stub->FetchTree(&client_context, request, &response);
        ASSERT_TRUE(fetchStatus.ok());

        // Neither the root directory nor the subdirectory was in the tree
        // cache
        ASSERT_TRUE(allCollectedByNameWithValues<CountingMetricValue>(
            {{MetricNames::COUNTER_NAME_LOCAL_CAS_TREE_CACHE_HITS,
              CountingMetricValue(0)},
             {MetricNames::COUNTER_NAME_LOCAL_CAS_TREE_CACHE_MISSES,
              CountingMetricValue(2)}}));

        // 0% tree cache hit rate
        ASSERT_TRUE(collectedByNameWithValue<DistributionMetricValue>(
            MetricNames::DISTRIBUTION_NAME_LOCAL_CAS_TREE_CACHE_HIT_PERCENTAGE,
            DistributionMetricValue(0)));
    }

    // Stage the fetched tree
    StageTreeRequest request;
    StageTreeResponse response;
    request.set_instance_name(instance_name);
    request.mutable_root_digest()->CopyFrom(root_directory_digest);
    auto reader_writer = localcas_stub->StageTree(&client_context);
    ASSERT_TRUE(reader_writer->Write(request));
    ASSERT_TRUE(reader_writer->Read(&response));
    // Unstage
    reader_writer->WritesDone();
    const auto stageStatus = reader_writer->Finish();
    ASSERT_EQ(stageStatus.error_code(), grpc::StatusCode::OK);

    ASSERT_TRUE(allCollectedByName<DurationMetricValue>(
        {{MetricNames::TIMER_NAME_LOCAL_CAS_STAGE_TREE_TOTAL},
         {MetricNames::TIMER_NAME_LOCAL_CAS_STAGE_TREE_PREPARE},
         {MetricNames::TIMER_NAME_LOCAL_CAS_STAGE_TREE_STAGE}}));

    // The root directory is in the tree cache due to fetching before staging
    ASSERT_TRUE(allCollectedByNameWithValues<CountingMetricValue>(
        {{MetricNames::COUNTER_NAME_LOCAL_CAS_TREE_CACHE_HITS,
          CountingMetricValue(1)},
         {MetricNames::COUNTER_NAME_LOCAL_CAS_TREE_CACHE_MISSES,
          CountingMetricValue(0)}}));

    // 100% tree cache hit rate in StageTree
    ASSERT_TRUE(collectedByNameWithValue<DistributionMetricValue>(
        MetricNames::DISTRIBUTION_NAME_LOCAL_CAS_TREE_CACHE_HIT_PERCENTAGE,
        DistributionMetricValue(100)));
}

TEST_P(RemoteServerFixture, StreamCloseCleansDirectory)
{
    StageTreeRequest request;
    StageTreeResponse response;

    request.set_instance_name(instance_name);
    request.mutable_root_digest()->CopyFrom(root_directory_digest);

    auto reader_writer = localcas_stub->StageTree(&client_context);
    ASSERT_TRUE(reader_writer->Write(request));
    ASSERT_TRUE(reader_writer->Read(&response));
    const std::string stage_path = response.path();
    ASSERT_TRUE(FileUtils::isDirectory(stage_path.c_str()));

    reader_writer->WritesDone();
    const auto status = reader_writer->Finish();
    ASSERT_EQ(status.error_code(), grpc::StatusCode::OK);
    ASSERT_FALSE(FileUtils::isDirectory(stage_path.c_str()));

    ASSERT_TRUE(allCollectedByName<DurationMetricValue>(
        {{MetricNames::TIMER_NAME_LOCAL_CAS_STAGE_TREE_TOTAL},
         {MetricNames::TIMER_NAME_LOCAL_CAS_STAGE_TREE_PREPARE},
         {MetricNames::TIMER_NAME_LOCAL_CAS_STAGE_TREE_STAGE}}));
}

TEST_P(RemoteServerFixture, GetLocalDiskUsage)
{
    /* This is the disk usage of the content generated by the fixture
     * (2 files and 2 directories).
     */
    const int64_t fixture_disk_usage = 321;

    const std::string extra_content = std::string(42, 'x');

    {
        /* Check initial disk usage */
        grpc::ClientContext client_context;
        GetLocalDiskUsageRequest request;
        GetLocalDiskUsageResponse response;
        localcas_stub->GetLocalDiskUsage(&client_context, request, &response);
        ASSERT_EQ(response.size_bytes(), fixture_disk_usage);
        ASSERT_EQ(response.quota_bytes(), 0);

        ASSERT_TRUE(collectedByName<DurationMetricValue>(
            MetricNames::TIMER_NAME_LOCAL_CAS_GET_LOCAL_DISK_USAGE));
    }

    {
        /* Add a file */
        grpc::ClientContext client_context;
        CaptureFilesRequest request;
        CaptureFilesResponse response;
        request.set_instance_name(instance_name);
        const buildboxcommon::TemporaryDirectory temp_dir1;
        auto file_path = write_file_to_existing_dir(
            std::string(temp_dir1.name()), extra_content, true);
        request.add_path(file_path);
        localcas_stub->CaptureFiles(&client_context, request, &response);
    }

    {
        /* Check disk usage again */
        grpc::ClientContext client_context;
        GetLocalDiskUsageRequest request;
        GetLocalDiskUsageResponse response;
        localcas_stub->GetLocalDiskUsage(&client_context, request, &response);
        ASSERT_EQ(response.size_bytes(),
                  fixture_disk_usage + extra_content.size());
        ASSERT_EQ(response.quota_bytes(), 0);
    }
}

class CasProxyFixture : public CasProxyModeFixture {
  protected:
    CasProxyFixture() {}

    Digest uploadTree(std::shared_ptr<LocalCas> storage)
    {
        generateAndStoreFiles(&file_contents, &file_digests, storage);
        return createDirectoryTree(file_digests, storage);
    }

    /* Sends a stage request for the given digest and verifies that the
     * contents that are made available match the files we uploaded
     */
    void stageAndAssertContentsMatch(const Digest &root_directory_digest)
    {
        // Preparing a (valid) request:
        StageTreeRequest request;
        request.set_instance_name(instance_name);
        request.mutable_root_digest()->CopyFrom(root_directory_digest);

        // Stage:
        auto reader_writer = localcas_stub->StageTree(&client_context);
        ASSERT_TRUE(reader_writer->Write(request));

        // The tree was staged and we got a valid path in the response:
        StageTreeResponse response;
        ASSERT_TRUE(reader_writer->Read(&response));
        const std::string stage_path = response.path();
        ASSERT_TRUE(FileUtils::isDirectory(stage_path.c_str()));

        assertStagedDirectoryContentsMatch(stage_path);

        // Ask the server to clean up:
        ASSERT_TRUE(reader_writer->Write(StageTreeRequest()));

        // Receive its last empty reply:
        ASSERT_TRUE(reader_writer->Read(&response));
        ASSERT_TRUE(response.path().empty());
        const auto status = reader_writer->Finish();
        ASSERT_EQ(status.error_code(), grpc::StatusCode::OK);

        ASSERT_FALSE(FileUtils::isDirectory(stage_path.c_str()));
    }

    void assertStagedDirectoryContentsMatch(const std::string &stage_root_path)
    {
        // The contents that were staged match the tree we requested:
        const std::string file1_stage_path = stage_root_path + "/file1.sh";
        ASSERT_TRUE(FileUtils::isRegularFile(file1_stage_path.c_str()));
        ASSERT_TRUE(FileUtils::isExecutable(file1_stage_path.c_str()));
        ASSERT_TRUE(
            fileContentMatches(file1_stage_path, file_contents["file1.sh"]));

        const std::string file2_stage_path =
            stage_root_path + "/subdir1/file2.c";
        ASSERT_TRUE(FileUtils::isRegularFile(file2_stage_path.c_str()));
        ASSERT_TRUE(
            fileContentMatches(file2_stage_path, file_contents["file2.c"]));

        const std::string link_stage_path = stage_root_path + "/link";
        ASSERT_TRUE(FileUtils::isRegularFile(link_stage_path.c_str()));
        ASSERT_TRUE(
            fileContentMatches(link_stage_path, file_contents["file2.c"]));
    }

    std::map<std::string, Digest> file_digests;
    std::map<std::string, std::string> file_contents;
};

TEST_P(CasProxyFixture, StageRootDigestNotInCAS)
{
    StageTreeRequest request;
    StageTreeResponse response;
    request.set_instance_name(instance_name);

    Digest d(DigestGenerator::hash(""));
    d.set_hash("hash1234");
    request.mutable_root_digest()->CopyFrom(d);

    auto reader_writer = localcas_stub->StageTree(&client_context);
    ASSERT_TRUE(reader_writer->Write(request));
    ASSERT_FALSE(reader_writer->Read(&response));

    const auto status = reader_writer->Finish();
    ASSERT_EQ(status.error_code(), grpc::StatusCode::FAILED_PRECONDITION);

    ASSERT_FALSE(collectedByName<DurationMetricValue>(
        MetricNames::TIMER_NAME_LOCAL_CAS_STAGE_TREE_STAGE));
}

TEST_P(CasProxyFixture, StageWithBlobsLocally)
{
    const Digest root_directory_digest = uploadTree(local_storage);

    // The tree is available locally:
    ASSERT_TRUE(local_storage->hasBlob(root_directory_digest));
    ASSERT_FALSE(remote_storage->hasBlob(root_directory_digest));

    stageAndAssertContentsMatch(root_directory_digest);
}

TEST_P(CasProxyFixture, StageWithBlobsInRemote)
{
    const Digest root_directory_digest = uploadTree(remote_storage);

    // The tree is not available locally, but the proxy fetches it from the
    // remote before staging:
    ASSERT_FALSE(local_storage->hasBlob(root_directory_digest));
    ASSERT_TRUE(remote_storage->hasBlob(root_directory_digest));

    stageAndAssertContentsMatch(root_directory_digest);
}

TEST_P(CasProxyFixture, FetchMissingBlobs)
{
    buildboxcommon::buildboxcommonmetrics::clearAllMetricCollection();
    const std::string blob1 = "Local Blob";
    const Digest digest1 = DigestGenerator::hash(blob1);
    local_storage->writeBlob(digest1, blob1);

    const std::string blob2 = "Remote Blob";
    const Digest digest2 = DigestGenerator::hash(blob2);
    remote_storage->writeBlob(digest2, blob2);
    const std::string blob3 = "Additional Remote Blob";
    const Digest digest3 = DigestGenerator::hash(blob3);
    remote_storage->writeBlob(digest3, blob3);

    ASSERT_TRUE(local_storage->hasBlob(digest1));
    ASSERT_FALSE(local_storage->hasBlob(digest2));
    ASSERT_FALSE(local_storage->hasBlob(digest3));

    FetchMissingBlobsRequest request;
    request.set_instance_name(instance_name);
    request.add_blob_digests()->CopyFrom(digest1);
    request.add_blob_digests()->CopyFrom(digest2);
    request.add_blob_digests()->CopyFrom(digest3);

    FetchMissingBlobsResponse response;

    const auto status =
        localcas_stub->FetchMissingBlobs(&client_context, request, &response);
    ASSERT_TRUE(status.ok());
    // This should trigger two remote reads, one for blob2 and one for
    // blob3 It will also trigger two "local" read in the remote_storage
    // when they are fetched.
    ASSERT_TRUE(allCollectedByNameWithValues<CountingMetricValue>(
        {{MetricNames::COUNTER_NUM_BLOBS_READ_FROM_REMOTE,
          CountingMetricValue(2)},
         {MetricNames::COUNTER_NUM_BLOBS_READ_FROM_LOCAL,
          CountingMetricValue(2)}}));

    ASSERT_TRUE(allCollectedByNameWithValues<DistributionMetricValue>({
        {MetricNames::DISTRIBUTION_NAME_CAS_FETCH_MISSING_BLOBS_SIZES,
         DistributionMetricValue(digest1.size_bytes())},
        {MetricNames::DISTRIBUTION_NAME_CAS_FETCH_MISSING_BLOBS_SIZES,
         DistributionMetricValue(digest2.size_bytes())},
        {MetricNames::DISTRIBUTION_NAME_CAS_FETCH_MISSING_BLOBS_SIZES,
         DistributionMetricValue(digest3.size_bytes())},
    }));

    ASSERT_EQ(response.responses().size(), 0);

    // `blob2` and `blob3` now available locally:
    ASSERT_TRUE(local_storage->hasBlob(digest2));
    ASSERT_TRUE(local_storage->hasBlob(digest3));
    ASSERT_EQ(readBlobFromLocalStorage(digest2), blob2);
    ASSERT_EQ(readBlobFromLocalStorage(digest3), blob3);
    ASSERT_TRUE(collectedByName<DurationMetricValue>(
        buildboxcasd::MetricNames::TIMER_NAME_LOCAL_CAS_FETCH_MISSING_BLOB));
    ASSERT_TRUE(
        allCollectedByNameWithValuesAndAllMissingByName<CountingMetricValue>(
            {{MetricNames::COUNTER_NUM_BLOBS_READ_FROM_LOCAL,
              CountingMetricValue(2)}},
            {MetricNames::COUNTER_NUM_BLOBS_READ_FROM_REMOTE}));
}

TEST_P(CasProxyFixture, FetchMissingBlobsCouldNotFetch)
{
    buildboxcommon::buildboxcommonmetrics::clearAllMetricCollection();
    const std::string blob = "BlobNotAvailable";
    const Digest digest = DigestGenerator::hash(blob);

    ASSERT_FALSE(local_storage->hasBlob(digest));
    ASSERT_FALSE(remote_storage->hasBlob(digest));

    FetchMissingBlobsRequest request;
    request.set_instance_name(instance_name);
    request.add_blob_digests()->CopyFrom(digest);

    FetchMissingBlobsResponse response;
    const auto status =
        localcas_stub->FetchMissingBlobs(&client_context, request, &response);
    ASSERT_TRUE(status.ok());

    ASSERT_EQ(response.responses().size(), 1);
    ASSERT_EQ(response.responses(0).digest(), digest);
    ASSERT_TRUE(collectedByName<DurationMetricValue>(
        buildboxcasd::MetricNames::TIMER_NAME_LOCAL_CAS_FETCH_MISSING_BLOB));
    // No metric count recorded.
    ASSERT_FALSE(collectedByName<CountingMetricValue>(
        MetricNames::COUNTER_NUM_BLOBS_READ_FROM_LOCAL));
    ASSERT_FALSE(collectedByName<CountingMetricValue>(
        MetricNames::COUNTER_NUM_BLOBS_READ_FROM_REMOTE));
}

TEST_P(CasProxyFixture, FetchMissingBlobsRequestRepeatedDigests)
{
    const std::string blob = "BlobNotAvailable";
    const Digest digest = DigestGenerator::hash(blob);

    ASSERT_FALSE(local_storage->hasBlob(digest));
    ASSERT_FALSE(remote_storage->hasBlob(digest));

    FetchMissingBlobsRequest request;
    request.set_instance_name(instance_name);
    request.add_blob_digests()->CopyFrom(digest);
    request.add_blob_digests()->CopyFrom(digest);
    request.add_blob_digests()->CopyFrom(digest);

    ASSERT_EQ(request.blob_digests_size(), 3);
    // The server should de-duplicate the entries and give us only one
    // response.

    FetchMissingBlobsResponse response;
    const auto status =
        localcas_stub->FetchMissingBlobs(&client_context, request, &response);
    ASSERT_TRUE(status.ok());

    ASSERT_EQ(response.responses().size(), 1);
    ASSERT_EQ(response.responses(0).digest(), digest);
}

TEST_P(CasProxyFixture, FetchTreeRemote)
{
    buildboxcommon::buildboxcommonmetrics::clearAllMetricCollection();
    const Digest root_directory_digest = uploadTree(remote_storage);

    // The tree is not available locally
    ASSERT_FALSE(local_storage->hasBlob(root_directory_digest));
    ASSERT_TRUE(remote_storage->hasBlob(root_directory_digest));

    FetchTreeRequest request;
    FetchTreeResponse response;
    request.set_instance_name(instance_name);
    request.mutable_root_digest()->CopyFrom(root_directory_digest);

    const auto status =
        localcas_stub->FetchTree(&client_context, request, &response);
    ASSERT_TRUE(status.ok());

    // We should read from the remote exactly how many digests there
    // are in the tree (the root and the subdir1)
    ASSERT_TRUE(allCollectedByNameWithValues<CountingMetricValue>({
        {MetricNames::COUNTER_NUM_BLOBS_READ_FROM_REMOTE,
         CountingMetricValue(2)},
        {MetricNames::COUNTER_NAME_LOCAL_CAS_NUM_FETCH_TREE_REQUESTS,
         CountingMetricValue(1)},
    }));

    // The tree is now available locally but without file blobs
    ASSERT_TRUE(local_storage->hasBlob(root_directory_digest));
    ASSERT_FALSE(local_storage->hasBlob(file_digests["file1.sh"]));
    ASSERT_TRUE(collectedByName<DurationMetricValue>(
        buildboxcasd::MetricNames::TIMER_NAME_LOCAL_CAS_FETCH_TREE));
}

TEST_P(CasProxyFixture, FetchTreeRemoteCachedResult)
{
    buildboxcommon::buildboxcommonmetrics::clearAllMetricCollection();
    const Digest root_directory_digest = uploadTree(remote_storage);

    // The tree is not available locally
    ASSERT_FALSE(local_storage->hasBlob(root_directory_digest));
    ASSERT_TRUE(remote_storage->hasBlob(root_directory_digest));

    FetchTreeRequest request;
    FetchTreeResponse response;
    request.set_instance_name(instance_name);
    request.mutable_root_digest()->CopyFrom(root_directory_digest);

    // Initial request. This will fetch the tree successfully to the local
    // storage:
    {
        grpc::ClientContext context;
        const auto status =
            localcas_stub->FetchTree(&context, request, &response);
        ASSERT_TRUE(status.ok());
        ASSERT_TRUE(local_storage->hasBlob(root_directory_digest));

        // We should read from the remote exactly how many digests there
        // are in the tree (the root and the subdir1)
        ASSERT_TRUE(allCollectedByNameWithValues<CountingMetricValue>({
            {MetricNames::COUNTER_NUM_BLOBS_READ_FROM_REMOTE,
             CountingMetricValue(2)},
            {MetricNames::COUNTER_NAME_LOCAL_CAS_NUM_FETCH_TREE_REQUESTS,
             CountingMetricValue(1)},
        }));
    }

    // We delete the blob from the storage and issue the same request.
    // This time `FetchTree()` makes a cache lookup, so it still returns
    // successfully.
    stopRemoteServer();
    local_storage->deleteBlob(root_directory_digest);
    {
        grpc::ClientContext context;
        const auto status =
            localcas_stub->FetchTree(&context, request, &response);
        ASSERT_TRUE(status.ok());
    }
}

TEST_P(CasProxyFixture, FetchTreeRemoteWithFiles)
{
    buildboxcommon::buildboxcommonmetrics::clearAllMetricCollection();
    const Digest root_directory_digest = uploadTree(remote_storage);

    // The tree is not available locally
    ASSERT_FALSE(local_storage->hasBlob(root_directory_digest));
    ASSERT_TRUE(remote_storage->hasBlob(root_directory_digest));

    FetchTreeRequest request;
    FetchTreeResponse response;
    request.set_instance_name(instance_name);
    request.mutable_root_digest()->CopyFrom(root_directory_digest);
    request.set_fetch_file_blobs(true);

    const auto status =
        localcas_stub->FetchTree(&client_context, request, &response);
    ASSERT_TRUE(status.ok());

    // Should have a count for every tree entry (2) and every blob (2)
    ASSERT_TRUE(allCollectedByNameWithValues<CountingMetricValue>({
        {MetricNames::COUNTER_NUM_BLOBS_READ_FROM_REMOTE,
         CountingMetricValue(4)},
        {MetricNames::COUNTER_NAME_LOCAL_CAS_NUM_FETCH_TREE_REQUESTS,
         CountingMetricValue(1)},
    }));

    // The tree is now available locally including file blobs
    ASSERT_TRUE(local_storage->hasBlob(root_directory_digest));
    ASSERT_TRUE(local_storage->hasBlob(file_digests["file1.sh"]));
}

TEST_P(CasProxyFixture, FetchTreeRemoteWithFilesCachedResult)
{
    buildboxcommon::buildboxcommonmetrics::clearAllMetricCollection();
    const Digest root_directory_digest = uploadTree(remote_storage);

    // The tree is not available locally
    ASSERT_FALSE(local_storage->hasBlob(root_directory_digest));
    ASSERT_TRUE(remote_storage->hasBlob(root_directory_digest));

    FetchTreeRequest request;
    FetchTreeResponse response;
    request.set_instance_name(instance_name);
    request.mutable_root_digest()->CopyFrom(root_directory_digest);
    request.set_fetch_file_blobs(true);

    {
        grpc::ClientContext context;
        const auto status =
            localcas_stub->FetchTree(&context, request, &response);
        ASSERT_TRUE(status.ok());

        // Should have a count for every tree entry (2) and every blob (2)
        ASSERT_TRUE(allCollectedByNameWithValues<CountingMetricValue>({
            {MetricNames::COUNTER_NUM_BLOBS_READ_FROM_REMOTE,
             CountingMetricValue(4)},
            {MetricNames::COUNTER_NAME_LOCAL_CAS_NUM_FETCH_TREE_REQUESTS,
             CountingMetricValue(1)},
        }));

        // The tree is now available locally including file blobs
        ASSERT_TRUE(local_storage->hasBlob(root_directory_digest));
        ASSERT_TRUE(local_storage->hasBlob(file_digests["file1.sh"]));
    }

    // We delete the blobs from the storage and issue the same request.
    // This time `FetchTree()` makes a cache lookup, so it still returns
    // successfully.
    stopRemoteServer();
    local_storage->deleteBlob(root_directory_digest);
    local_storage->deleteBlob(file_digests["file1.sh"]);

    {
        grpc::ClientContext context;
        const auto status =
            localcas_stub->FetchTree(&context, request, &response);
        ASSERT_TRUE(status.ok());

        ASSERT_TRUE(collectedByNameWithValue(
            MetricNames::COUNTER_NAME_LOCAL_CAS_NUM_FETCH_TREE_CACHE_HITS,
            CountingMetricValue(1)));
    }
}

TEST_P(CasProxyFixture, FetchTreeNotInCAS)
{
    const Digest root_directory_digest =
        DigestGenerator::hash("does not exist");

    // The tree does not exist
    ASSERT_FALSE(local_storage->hasBlob(root_directory_digest));
    ASSERT_FALSE(remote_storage->hasBlob(root_directory_digest));

    FetchTreeRequest request;
    FetchTreeResponse response;
    request.set_instance_name(instance_name);
    request.mutable_root_digest()->CopyFrom(root_directory_digest);

    const auto status =
        localcas_stub->FetchTree(&client_context, request, &response);
    ASSERT_EQ(status.error_code(), grpc::StatusCode::NOT_FOUND);
    ASSERT_TRUE(collectedByName<DurationMetricValue>(
        buildboxcasd::MetricNames::TIMER_NAME_LOCAL_CAS_FETCH_TREE));
}

TEST_P(CasProxyFixture, FetchTreeLocal)
{
    const Digest root_directory_digest = uploadTree(local_storage);

    // The tree is already available locally
    ASSERT_TRUE(local_storage->hasBlob(root_directory_digest));
    ASSERT_FALSE(remote_storage->hasBlob(root_directory_digest));

    FetchTreeRequest request;
    FetchTreeResponse response;
    request.set_instance_name(instance_name);
    request.mutable_root_digest()->CopyFrom(root_directory_digest);

    const auto status =
        localcas_stub->FetchTree(&client_context, request, &response);
    ASSERT_TRUE(status.ok());

    // Neither the root directory nor the subdirectory was in the tree cache
    ASSERT_TRUE(allCollectedByNameWithValues<CountingMetricValue>(
        {{MetricNames::COUNTER_NAME_LOCAL_CAS_TREE_CACHE_HITS,
          CountingMetricValue(0)},
         {MetricNames::COUNTER_NAME_LOCAL_CAS_TREE_CACHE_MISSES,
          CountingMetricValue(2)}}));

    // 0% tree cache hit rate
    ASSERT_TRUE(collectedByNameWithValue<DistributionMetricValue>(
        MetricNames::DISTRIBUTION_NAME_LOCAL_CAS_TREE_CACHE_HIT_PERCENTAGE,
        DistributionMetricValue(0)));
}

TEST_P(CasProxyFixture, UploadTree)
{
    const Digest root_directory_digest = uploadTree(local_storage);

    // The tree is available locally but not on the remote
    ASSERT_TRUE(local_storage->hasBlob(root_directory_digest));
    ASSERT_FALSE(remote_storage->hasBlob(root_directory_digest));

    UploadTreeRequest request;
    UploadTreeResponse response;
    request.set_instance_name(instance_name);
    request.mutable_root_digest()->CopyFrom(root_directory_digest);

    const auto status =
        localcas_stub->UploadTree(&client_context, request, &response);
    ASSERT_TRUE(status.ok());

    // Now the tree is on the remote including file blobs
    ASSERT_TRUE(remote_storage->hasBlob(root_directory_digest));
    ASSERT_TRUE(remote_storage->hasBlob(file_digests["file1.sh"]));
}

TEST_P(CasProxyFixture, UploadTreeMissingRemoteBlob)
{
    const Digest root_directory_digest = uploadTree(local_storage);

    // Let the remote already have the tree as well but missing a file blob
    uploadTree(remote_storage);
    remote_storage->deleteBlob(file_digests["file1.sh"]);

    ASSERT_TRUE(local_storage->hasBlob(root_directory_digest));
    ASSERT_TRUE(remote_storage->hasBlob(root_directory_digest));
    ASSERT_FALSE(remote_storage->hasBlob(file_digests["file1.sh"]));

    UploadTreeRequest request;
    UploadTreeResponse response;
    request.set_instance_name(instance_name);
    request.mutable_root_digest()->CopyFrom(root_directory_digest);

    const auto status =
        localcas_stub->UploadTree(&client_context, request, &response);
    ASSERT_TRUE(status.ok());

    // Now the tree is on the remote including file blobs
    ASSERT_TRUE(remote_storage->hasBlob(root_directory_digest));
    ASSERT_TRUE(remote_storage->hasBlob(file_digests["file1.sh"]));
}

TEST_P(CasProxyFixture, UploadTreeMissingLocalBlob)
{
    const Digest root_directory_digest = uploadTree(local_storage);
    uploadTree(remote_storage);

    // A file blob missing in the local cache should not result in a failure
    // as long as the file blob is on the remote.
    local_storage->deleteBlob(file_digests["file1.sh"]);

    ASSERT_TRUE(local_storage->hasBlob(root_directory_digest));
    ASSERT_FALSE(local_storage->hasBlob(file_digests["file1.sh"]));
    ASSERT_TRUE(remote_storage->hasBlob(root_directory_digest));
    ASSERT_TRUE(remote_storage->hasBlob(file_digests["file1.sh"]));

    UploadTreeRequest request;
    UploadTreeResponse response;
    request.set_instance_name(instance_name);
    request.mutable_root_digest()->CopyFrom(root_directory_digest);

    const auto status =
        localcas_stub->UploadTree(&client_context, request, &response);
    ASSERT_TRUE(status.ok());

    // The file blob will still be missing from the local cache
    ASSERT_TRUE(local_storage->hasBlob(root_directory_digest));
    ASSERT_FALSE(local_storage->hasBlob(file_digests["file1.sh"]));
}

TEST_P(CasProxyFixture, UploadTreeMissingDirectoryBlob)
{
    const Digest root_directory_digest = uploadTree(local_storage);

    // If a directory blob is missing locally, UploadTree should fail
    local_storage->deleteBlob(root_directory_digest);

    ASSERT_FALSE(local_storage->hasBlob(root_directory_digest));
    ASSERT_FALSE(remote_storage->hasBlob(root_directory_digest));

    UploadTreeRequest request;
    UploadTreeResponse response;
    request.set_instance_name(instance_name);
    request.mutable_root_digest()->CopyFrom(root_directory_digest);

    const auto status =
        localcas_stub->UploadTree(&client_context, request, &response);
    ASSERT_FALSE(status.ok());
    ASSERT_EQ(status.error_code(), grpc::StatusCode::NOT_FOUND);
}

TEST_P(CasProxyFixture, UploadTreeMissingFileBlob)
{
    const Digest root_directory_digest = uploadTree(local_storage);
    uploadTree(remote_storage);

    // If a file blob is missing locally and remotely, UploadTree should fail
    local_storage->deleteBlob(file_digests["file1.sh"]);
    remote_storage->deleteBlob(file_digests["file1.sh"]);

    ASSERT_TRUE(local_storage->hasBlob(root_directory_digest));
    ASSERT_FALSE(local_storage->hasBlob(file_digests["file1.sh"]));
    ASSERT_TRUE(remote_storage->hasBlob(root_directory_digest));
    ASSERT_FALSE(remote_storage->hasBlob(file_digests["file1.sh"]));

    UploadTreeRequest request;
    UploadTreeResponse response;
    request.set_instance_name(instance_name);
    request.mutable_root_digest()->CopyFrom(root_directory_digest);

    const auto status =
        localcas_stub->UploadTree(&client_context, request, &response);
    ASSERT_FALSE(status.ok());
    ASSERT_EQ(status.error_code(), grpc::StatusCode::NOT_FOUND);
}

class CasProxyUnreachableRemoteFixture : public CasProxyFixture {
  protected:
    CasProxyUnreachableRemoteFixture() { stopRemoteServer(); }
};

TEST_P(CasProxyUnreachableRemoteFixture,
       FetchMissingBlobsWithUnreachableRemote)
{
    const std::string blob = "Some blob";
    const Digest digest = DigestGenerator::hash(blob);

    FetchMissingBlobsRequest request;
    request.set_instance_name(instance_name);
    request.add_blob_digests()->CopyFrom(digest);

    FetchMissingBlobsResponse response;

    const auto status =
        localcas_stub->FetchMissingBlobs(&client_context, request, &response);
    ASSERT_TRUE(status.ok());

    ASSERT_EQ(response.responses_size(), 1);
    ASSERT_EQ(response.responses(0).digest(), digest);
    ASSERT_NE(response.responses(0).status().code(), grpc::StatusCode::OK);
}

TEST_P(CasProxyUnreachableRemoteFixture, FetchTreeRemoteWithUnreachableRemote)
{
    const Digest root_directory_digest = uploadTree(remote_storage);

    // The tree is not available locally
    ASSERT_FALSE(local_storage->hasBlob(root_directory_digest));
    ASSERT_TRUE(remote_storage->hasBlob(root_directory_digest));

    FetchTreeRequest request;
    FetchTreeResponse response;
    request.set_instance_name(instance_name);
    request.mutable_root_digest()->CopyFrom(root_directory_digest);

    const auto status =
        localcas_stub->FetchTree(&client_context, request, &response);
    ASSERT_FALSE(status.ok());
}

#endif // INCLUDED_BUILDBOXCASD_LOCALCASSERVICE_H
