mctaylors/suyu
1
0
Fork 0
forked from suyu/suyu
Code Pull requests Activity

bcat: Implement DeliveryCacheProgressImpl structure

Browse source 
Huge thanks to lioncash for re-ing this for me.
This commit is contained in:
Zach Hilman 2019-05-13 18:51:02 -04:00
parent 92b70a3bf9
commit 2d410ddf4d
6 changed files with 314 additions and 88 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
src/core/file_sys/vfs_libzip.cpp
View file

@ -15,13 +15,13 @@ VirtualDir ExtractZIP(VirtualFile file) {
zip_error_t error{};
const auto data = file->ReadAllBytes();
std::unique_ptr<zip_source_t, decltype(&zip_source_free)> src{
zip_source_buffer_create(data.data(), data.size(), 0, &error), zip_source_free};
std::unique_ptr<zip_source_t, decltype(&zip_source_close)> src{
zip_source_buffer_create(data.data(), data.size(), 0, &error), zip_source_close};
if (src == nullptr)
return nullptr;
std::unique_ptr<zip_t, decltype(&zip_discard)> zip{zip_open_from_source(src.get(), 0, &error),
zip_discard};
std::unique_ptr<zip_t, decltype(&zip_close)> zip{zip_open_from_source(src.get(), 0, &error),
zip_close};
if (zip == nullptr)
return nullptr;

88
src/core/hle/service/bcat/backend/backend.cpp
View file

@ -4,10 +4,90 @@
#include "common/hex_util.h"
#include "common/logging/log.h"
#include "core/core.h"
#include "core/hle/lock.h"
#include "core/hle/service/bcat/backend/backend.h"
namespace Service::BCAT {
ProgressServiceBackend::ProgressServiceBackend(std::string event_name) : impl{} {
auto& kernel{Core::System::GetInstance().Kernel()};
event = Kernel::WritableEvent::CreateEventPair(
kernel, Kernel::ResetType::OneShot, "ProgressServiceBackend:UpdateEvent:" + event_name);
}
Kernel::SharedPtr<Kernel::ReadableEvent> ProgressServiceBackend::GetEvent() {
return event.readable;
}
DeliveryCacheProgressImpl& ProgressServiceBackend::GetImpl() {
return impl;
}
void ProgressServiceBackend::SetNeedHLELock(bool need) {
need_hle_lock = need;
}
void ProgressServiceBackend::SetTotalSize(u64 size) {
impl.total_bytes = size;
SignalUpdate();
}
void ProgressServiceBackend::StartConnecting() {
impl.status = DeliveryCacheProgressImpl::Status::Connecting;
SignalUpdate();
}
void ProgressServiceBackend::StartProcessingDataList() {
impl.status = DeliveryCacheProgressImpl::Status::ProcessingDataList;
SignalUpdate();
}
void ProgressServiceBackend::StartDownloadingFile(std::string_view dir_name,
std::string_view file_name, u64 file_size) {
impl.status = DeliveryCacheProgressImpl::Status::Downloading;
impl.current_downloaded_bytes = 0;
impl.current_total_bytes = file_size;
std::memcpy(impl.current_directory.data(), dir_name.data(), std::min(dir_name.size(), 0x31ull));
std::memcpy(impl.current_file.data(), file_name.data(), std::min(file_name.size(), 0x31ull));
SignalUpdate();
}
void ProgressServiceBackend::UpdateFileProgress(u64 downloaded) {
impl.current_downloaded_bytes = downloaded;
SignalUpdate();
}
void ProgressServiceBackend::FinishDownloadingFile() {
impl.total_downloaded_bytes += impl.current_total_bytes;
SignalUpdate();
}
void ProgressServiceBackend::CommitDirectory(std::string_view dir_name) {
impl.status = DeliveryCacheProgressImpl::Status::Committing;
impl.current_file.fill(0);
impl.current_downloaded_bytes = 0;
impl.current_total_bytes = 0;
std::memcpy(impl.current_directory.data(), dir_name.data(), std::min(dir_name.size(), 0x31ull));
SignalUpdate();
}
void ProgressServiceBackend::FinishDownload(ResultCode result) {
impl.total_downloaded_bytes = impl.total_bytes;
impl.status = DeliveryCacheProgressImpl::Status::Done;
impl.result = result;
SignalUpdate();
}
void ProgressServiceBackend::SignalUpdate() const {
if (need_hle_lock) {
std::lock_guard<std::recursive_mutex> lock(HLE::g_hle_lock);
event.writable->Signal();
} else {
event.writable->Signal();
}
}
Backend::Backend(DirectoryGetter getter) : dir_getter(std::move(getter)) {}
Backend::~Backend() = default;
@ -16,20 +96,20 @@ NullBackend::NullBackend(const DirectoryGetter& getter) : Backend(std::move(gett
NullBackend::~NullBackend() = default;
bool NullBackend::Synchronize(TitleIDVersion title, CompletionCallback callback) {
bool NullBackend::Synchronize(TitleIDVersion title, ProgressServiceBackend& progress) {
LOG_DEBUG(Service_BCAT, "called, title_id={:016X}, build_id={:016X}", title.title_id,
title.build_id);
callback(true);
progress.FinishDownload(RESULT_SUCCESS);
return true;
}
bool NullBackend::SynchronizeDirectory(TitleIDVersion title, std::string name,
CompletionCallback callback) {
ProgressServiceBackend& progress) {
LOG_DEBUG(Service_BCAT, "called, title_id={:016X}, build_id={:016X}, name={}", title.title_id,
title.build_id, name);
callback(true);
progress.FinishDownload(RESULT_SUCCESS);
return true;
}

97
src/core/hle/service/bcat/backend/backend.h
View file

@ -8,10 +8,14 @@
#include <optional>
#include "common/common_types.h"
#include "core/file_sys/vfs_types.h"
#include "core/hle/kernel/readable_event.h"
#include "core/hle/kernel/writable_event.h"
#include "core/hle/result.h"
namespace Service::BCAT {
using CompletionCallback = std::function<void(bool)>;
struct DeliveryCacheProgressImpl;
using DirectoryGetter = std::function<FileSys::VirtualDir(u64)>;
using Passphrase = std::array<u8, 0x20>;
@ -20,33 +24,116 @@ struct TitleIDVersion {
u64 build_id;
};
using DirectoryName = std::array<char, 0x20>;
using FileName = std::array<char, 0x20>;
struct DeliveryCacheProgressImpl {
enum class Status : s32 {
None = 0x0,
Queued = 0x1,
Connecting = 0x2,
ProcessingDataList = 0x3,
Downloading = 0x4,
Committing = 0x5,
Done = 0x9,
};
Status status;
ResultCode result = RESULT_SUCCESS;
DirectoryName current_directory;
FileName current_file;
s64 current_downloaded_bytes; ///< Bytes downloaded on current file.
s64 current_total_bytes; ///< Bytes total on current file.
s64 total_downloaded_bytes; ///< Bytes downloaded on overall download.
s64 total_bytes; ///< Bytes total on overall download.
INSERT_PADDING_BYTES(
0x198); ///< Appears to be unused in official code, possibly reserved for future use.
};
static_assert(sizeof(DeliveryCacheProgressImpl) == 0x200,
"DeliveryCacheProgressImpl has incorrect size.");
// A class to manage the signalling to the game about BCAT download progress.
// Some of this class is implemented in module.cpp to avoid exposing the implementation structure.
class ProgressServiceBackend {
friend class IBcatService;
ProgressServiceBackend(std::string event_name);
Kernel::SharedPtr<Kernel::ReadableEvent> GetEvent();
DeliveryCacheProgressImpl& GetImpl();
public:
// Clients should call this with true if any of the functions are going to be called from a
// non-HLE thread and this class need to lock the hle mutex. (default is false)
void SetNeedHLELock(bool need);
// Sets the number of bytes total in the entire download.
void SetTotalSize(u64 size);
// Notifies the application that the backend has started connecting to the server.
void StartConnecting();
// Notifies the application that the backend has begun accumulating and processing metadata.
void StartProcessingDataList();
// Notifies the application that a file is starting to be downloaded.
void StartDownloadingFile(std::string_view dir_name, std::string_view file_name, u64 file_size);
// Updates the progress of the current file to the size passed.
void UpdateFileProgress(u64 downloaded);
// Notifies the application that the current file has completed download.
void FinishDownloadingFile();
// Notifies the application that all files in this directory have completed and are being
// finalized.
void CommitDirectory(std::string_view dir_name);
// Notifies the application that the operation completed with result code result.
void FinishDownload(ResultCode result);
private:
void SignalUpdate() const;
DeliveryCacheProgressImpl impl;
Kernel::EventPair event;
bool need_hle_lock = false;
};
// A class representing an abstract backend for BCAT functionality.
class Backend {
public:
explicit Backend(DirectoryGetter getter);
virtual ~Backend();
virtual bool Synchronize(TitleIDVersion title, CompletionCallback callback) = 0;
// Called when the backend is needed to synchronize the data for the game with title ID and
// version in title. A ProgressServiceBackend object is provided to alert the application of
// status.
virtual bool Synchronize(TitleIDVersion title, ProgressServiceBackend& progress) = 0;
// Very similar to Synchronize, but only for the directory provided. Backends should not alter
// the data for any other directories.
virtual bool SynchronizeDirectory(TitleIDVersion title, std::string name,
CompletionCallback callback) = 0;
ProgressServiceBackend& progress) = 0;
// Removes all cached data associated with title id provided.
virtual bool Clear(u64 title_id) = 0;
// Sets the BCAT Passphrase to be used with the associated title ID.
virtual void SetPassphrase(u64 title_id, const Passphrase& passphrase) = 0;
// Gets the launch parameter used by AM associated with the title ID and version provided.
virtual std::optional<std::vector<u8>> GetLaunchParameter(TitleIDVersion title) = 0;
protected:
DirectoryGetter dir_getter;
};
// A backend of BCAT that provides no operation.
class NullBackend : public Backend {
public:
explicit NullBackend(const DirectoryGetter& getter);
~NullBackend() override;
bool Synchronize(TitleIDVersion title, CompletionCallback callback) override;
bool Synchronize(TitleIDVersion title, ProgressServiceBackend& progress) override;
bool SynchronizeDirectory(TitleIDVersion title, std::string name,
CompletionCallback callback) override;
ProgressServiceBackend& progress) override;
bool Clear(u64 title_id) override;

147
src/core/hle/service/bcat/backend/boxcat.cpp
View file

@ -14,13 +14,28 @@
#include "core/file_sys/vfs_libzip.h"
#include "core/file_sys/vfs_vector.h"
#include "core/frontend/applets/error.h"
#include "core/hle/lock.h"
#include "core/hle/service/am/applets/applets.h"
#include "core/hle/service/bcat/backend/boxcat.h"
#include "core/settings.h"
namespace {
// Prevents conflicts with windows macro called CreateFile
FileSys::VirtualFile VfsCreateFileWrap(FileSys::VirtualDir dir, std::string_view name) {
return dir->CreateFile(name);
}
// Prevents conflicts with windows macro called DeleteFile
bool VfsDeleteFileWrap(FileSys::VirtualDir dir, std::string_view name) {
return dir->DeleteFile(name);
}
} // Anonymous namespace
namespace Service::BCAT {
constexpr ResultCode ERROR_GENERAL_BCAT_FAILURE{ErrorModule::BCAT, 1};
constexpr char BOXCAT_HOSTNAME[] = "api.yuzu-emu.org";
// Formatted using fmt with arg[0] = hex title id
@ -102,7 +117,68 @@ void HandleDownloadDisplayResult(DownloadResult res) {
DOWNLOAD_RESULT_LOG_MESSAGES[static_cast<std::size_t>(res)], [] {});
}
} // namespace
bool VfsRawCopyProgress(FileSys::VirtualFile src, FileSys::VirtualFile dest,
std::string_view dir_name, ProgressServiceBackend& progress,
std::size_t block_size = 0x1000) {
if (src == nullptr || dest == nullptr || !src->IsReadable() || !dest->IsWritable())
return false;
if (!dest->Resize(src->GetSize()))
return false;
progress.StartDownloadingFile(dir_name, src->GetName(), src->GetSize());
std::vector<u8> temp(std::min(block_size, src->GetSize()));
for (std::size_t i = 0; i < src->GetSize(); i += block_size) {
const auto read = std::min(block_size, src->GetSize() - i);
if (src->Read(temp.data(), read, i) != read) {
return false;
}
if (dest->Write(temp.data(), read, i) != read) {
return false;
}
progress.UpdateFileProgress(i);
}
progress.FinishDownloadingFile();
return true;
}
bool VfsRawCopyDProgressSingle(FileSys::VirtualDir src, FileSys::VirtualDir dest,
ProgressServiceBackend& progress, std::size_t block_size = 0x1000) {
if (src == nullptr || dest == nullptr || !src->IsReadable() || !dest->IsWritable())
return false;
for (const auto& file : src->GetFiles()) {
const auto out_file = VfsCreateFileWrap(dest, file->GetName());
if (!VfsRawCopyProgress(file, out_file, src->GetName(), progress, block_size)) {
return false;
}
}
progress.CommitDirectory(src->GetName());
return true;
}
bool VfsRawCopyDProgress(FileSys::VirtualDir src, FileSys::VirtualDir dest,
ProgressServiceBackend& progress, std::size_t block_size = 0x1000) {
if (src == nullptr || dest == nullptr || !src->IsReadable() || !dest->IsWritable())
return false;
for (const auto& dir : src->GetSubdirectories()) {
const auto out = dest->CreateSubdirectory(dir->GetName());
if (!VfsRawCopyDProgressSingle(dir, out, progress, block_size)) {
return false;
}
}
return true;
}
} // Anonymous namespace
class Boxcat::Client {
public:
@ -194,24 +270,24 @@ Boxcat::Boxcat(DirectoryGetter getter) : Backend(std::move(getter)) {}
Boxcat::~Boxcat() = default;
void SynchronizeInternal(DirectoryGetter dir_getter, TitleIDVersion title,
CompletionCallback callback, std::optional<std::string> dir_name = {}) {
const auto failure = [&callback] {
// Acquire the HLE mutex
std::lock_guard lock{HLE::g_hle_lock};
callback(false);
};
ProgressServiceBackend& progress,
std::optional<std::string> dir_name = {}) {
progress.SetNeedHLELock(true);
if (Settings::values.bcat_boxcat_local) {
LOG_INFO(Service_BCAT, "Boxcat using local data by override, skipping download.");
// Acquire the HLE mutex
std::lock_guard lock{HLE::g_hle_lock};
callback(true);
const auto dir = dir_getter(title.title_id);
if (dir)
progress.SetTotalSize(dir->GetSize());
progress.FinishDownload(RESULT_SUCCESS);
return;
}
const auto zip_path{GetZIPFilePath(title.title_id)};
Boxcat::Client client{zip_path, title.title_id, title.build_id};
progress.StartConnecting();
const auto res = client.DownloadDataZip();
if (res != DownloadResult::Success) {
LOG_ERROR(Service_BCAT, "Boxcat synchronization failed with error '{}'!", res);
@ -221,68 +297,85 @@ void SynchronizeInternal(DirectoryGetter dir_getter, TitleIDVersion title,
}
HandleDownloadDisplayResult(res);
failure();
progress.FinishDownload(ERROR_GENERAL_BCAT_FAILURE);
return;
}
progress.StartProcessingDataList();
FileUtil::IOFile zip{zip_path, "rb"};
const auto size = zip.GetSize();
std::vector<u8> bytes(size);
if (!zip.IsOpen() || size == 0 || zip.ReadBytes(bytes.data(), bytes.size()) != bytes.size()) {
LOG_ERROR(Service_BCAT, "Boxcat failed to read ZIP file at path '{}'!", zip_path);
failure();
progress.FinishDownload(ERROR_GENERAL_BCAT_FAILURE);
return;
}
const auto extracted = FileSys::ExtractZIP(std::make_shared<FileSys::VectorVfsFile>(bytes));
if (extracted == nullptr) {
LOG_ERROR(Service_BCAT, "Boxcat failed to extract ZIP file!");
failure();
progress.FinishDownload(ERROR_GENERAL_BCAT_FAILURE);
return;
}
if (dir_name == std::nullopt) {
progress.SetTotalSize(extracted->GetSize());
const auto target_dir = dir_getter(title.title_id);
if (target_dir == nullptr ||
!FileSys::VfsRawCopyD(extracted, target_dir, VFS_COPY_BLOCK_SIZE)) {
if (target_dir == nullptr || !VfsRawCopyDProgress(extracted, target_dir, progress)) {
LOG_ERROR(Service_BCAT, "Boxcat failed to copy extracted ZIP to target directory!");
failure();
progress.FinishDownload(ERROR_GENERAL_BCAT_FAILURE);
return;
}
} else {
const auto target_dir = dir_getter(title.title_id);
if (target_dir == nullptr) {
LOG_ERROR(Service_BCAT, "Boxcat failed to get directory for title ID!");
failure();
progress.FinishDownload(ERROR_GENERAL_BCAT_FAILURE);
return;
}
const auto target_sub = target_dir->GetSubdirectory(*dir_name);
const auto source_sub = extracted->GetSubdirectory(*dir_name);
progress.SetTotalSize(source_sub->GetSize());
std::vector<std::string> filenames;
{
const auto files = target_sub->GetFiles();
std::transform(files.begin(), files.end(), std::back_inserter(filenames),
[](const auto& vfile) { return vfile->GetName(); });
}
for (const auto& filename : filenames) {
VfsDeleteFileWrap(target_sub, filename);
}
if (target_sub == nullptr || source_sub == nullptr ||
!FileSys::VfsRawCopyD(source_sub, target_sub, VFS_COPY_BLOCK_SIZE)) {
!VfsRawCopyDProgressSingle(source_sub, target_sub, progress)) {
LOG_ERROR(Service_BCAT, "Boxcat failed to copy extracted ZIP to target directory!");
failure();
progress.FinishDownload(ERROR_GENERAL_BCAT_FAILURE);
return;
}
}
// Acquire the HLE mutex
std::lock_guard lock{HLE::g_hle_lock};
callback(true);
progress.FinishDownload(RESULT_SUCCESS);
}
bool Boxcat::Synchronize(TitleIDVersion title, CompletionCallback callback) {
bool Boxcat::Synchronize(TitleIDVersion title, ProgressServiceBackend& progress) {
is_syncing.exchange(true);
std::thread(&SynchronizeInternal, dir_getter, title, callback, std::nullopt).detach();
std::thread([this, title, &progress] { SynchronizeInternal(dir_getter, title, progress); })
.detach();
return true;
}
bool Boxcat::SynchronizeDirectory(TitleIDVersion title, std::string name,
CompletionCallback callback) {
ProgressServiceBackend& progress) {
is_syncing.exchange(true);
std::thread(&SynchronizeInternal, dir_getter, title, callback, name).detach();
std::thread(
[this, title, name, &progress] { SynchronizeInternal(dir_getter, title, progress, name); })
.detach();
return true;
}

6
src/core/hle/service/bcat/backend/boxcat.h
View file

@ -21,16 +21,16 @@ struct EventStatus {
/// doesn't require a switch or nintendo account. The content is controlled by the yuzu team.
class Boxcat final : public Backend {
friend void SynchronizeInternal(DirectoryGetter dir_getter, TitleIDVersion title,
CompletionCallback callback,
ProgressServiceBackend& progress,
std::optional<std::string> dir_name);
public:
explicit Boxcat(DirectoryGetter getter);
~Boxcat() override;
bool Synchronize(TitleIDVersion title, CompletionCallback callback) override;
bool Synchronize(TitleIDVersion title, ProgressServiceBackend& progress) override;
bool SynchronizeDirectory(TitleIDVersion title, std::string name,
CompletionCallback callback) override;
ProgressServiceBackend& progress) override;
bool Clear(u64 title_id) override;

56
src/core/hle/service/bcat/module.cpp
View file

@ -33,20 +33,6 @@ constexpr ResultCode ERROR_FAILED_CLEAR_CACHE{ErrorModule::FS, 6400};
using BCATDigest = std::array<u8, 0x10>;
struct DeliveryCacheProgressImpl {
enum class Status : u8 {
Incomplete = 0x1,
Complete = 0x9,
};
Status status = Status::Incomplete;
INSERT_PADDING_BYTES(
0x1FF); ///< TODO(DarkLordZach): RE this structure. It just seems to convey info about the
///< progress of the BCAT sync, but for us just setting completion works.
};
static_assert(sizeof(DeliveryCacheProgressImpl) == 0x200,
"DeliveryCacheProgressImpl has incorrect size.");
namespace {
u64 GetCurrentBuildID() {
@ -84,19 +70,16 @@ bool VerifyNameValidInternal(Kernel::HLERequestContext& ctx, std::array<char, 0x
return true;
}
bool VerifyNameValidDir(Kernel::HLERequestContext& ctx, std::array<char, 0x20> name) {
bool VerifyNameValidDir(Kernel::HLERequestContext& ctx, DirectoryName name) {
return VerifyNameValidInternal(ctx, name, '-');
}
bool VerifyNameValidFile(Kernel::HLERequestContext& ctx, std::array<char, 0x20> name) {
bool VerifyNameValidFile(Kernel::HLERequestContext& ctx, FileName name) {
return VerifyNameValidInternal(ctx, name, '.');
}
} // Anonymous namespace
using DirectoryName = std::array<char, 0x20>;
using FileName = std::array<char, 0x20>;
struct DeliveryCacheDirectoryEntry {
FileName name;
u64 size;
@ -162,15 +145,6 @@ public:
};
// clang-format on
RegisterHandlers(functions);
auto& kernel{Core::System::GetInstance().Kernel()};
progress.at(static_cast<std::size_t>(SyncType::Normal)).event =
Kernel::WritableEvent::CreateEventPair(kernel, Kernel::ResetType::Sticky,
"BCAT::IDeliveryCacheProgressEvent");
progress.at(static_cast<std::size_t>(SyncType::Directory)).event =
Kernel::WritableEvent::CreateEventPair(
kernel, Kernel::ResetType::OneShot,
"BCAT::IDeliveryCacheProgressEvent::DirectoryName");
}
private:
@ -180,24 +154,17 @@ private:
Count,
};
std::function<void(bool)> CreateCallback(SyncType type) {
return [this, type](bool success) {
auto& pair{progress.at(static_cast<std::size_t>(type))};
pair.impl.status = DeliveryCacheProgressImpl::Status::Complete;
pair.event.writable->Signal();
};
}
std::shared_ptr<IDeliveryCacheProgressService> CreateProgressService(SyncType type) {
const auto& pair{progress.at(static_cast<std::size_t>(type))};
return std::make_shared<IDeliveryCacheProgressService>(pair.event.readable, pair.impl);
auto& backend{progress.at(static_cast<std::size_t>(type))};
return std::make_shared<IDeliveryCacheProgressService>(backend.GetEvent(),
backend.GetImpl());
}
void RequestSyncDeliveryCache(Kernel::HLERequestContext& ctx) {
LOG_DEBUG(Service_BCAT, "called");
backend.Synchronize({Core::CurrentProcess()->GetTitleID(), GetCurrentBuildID()},
CreateCallback(SyncType::Normal));
progress.at(static_cast<std::size_t>(SyncType::Normal)));
IPC::ResponseBuilder rb{ctx, 2, 0, 1};
rb.Push(RESULT_SUCCESS);
@ -213,7 +180,8 @@ private:
LOG_DEBUG(Service_BCAT, "called, name={}", name);
backend.SynchronizeDirectory({Core::CurrentProcess()->GetTitleID(), GetCurrentBuildID()},
name, CreateCallback(SyncType::Directory));
name,
progress.at(static_cast<std::size_t>(SyncType::Directory)));
IPC::ResponseBuilder rb{ctx, 2, 0, 1};
rb.Push(RESULT_SUCCESS);
@ -278,12 +246,10 @@ private:
Backend& backend;
struct ProgressPair {
Kernel::EventPair event;
DeliveryCacheProgressImpl impl;
std::array<ProgressServiceBackend, static_cast<std::size_t>(SyncType::Count)> progress{
ProgressServiceBackend{"Normal"},
ProgressServiceBackend{"Directory"},
};
std::array<ProgressPair, static_cast<std::size_t>(SyncType::Count)> progress{};
};
void Module::Interface::CreateBcatService(Kernel::HLERequestContext& ctx) {