init

Telegram/lib_storage/storage/cache/storage_cache_binlog_reader.cpp

@@ -0,0 +1,110 @@
+// This file is part of Desktop App Toolkit,
+// a set of libraries for developing nice desktop applications.
+//
+// For license and copyright information please follow this link:
+// https://github.com/desktop-app/legal/blob/master/LEGAL
+//
+#include "storage/cache/storage_cache_binlog_reader.h"
+
+namespace Storage {
+namespace Cache {
+namespace details {
+
+BinlogWrapper::BinlogWrapper(
+	File &binlog,
+	const Settings &settings,
+	int64 till)
+: _binlog(binlog)
+, _settings(settings)
+, _till(till ? till : _binlog.size())
+, _data(_settings.readBlockSize)
+, _full(_data) {
+}
+
+bool BinlogWrapper::finished() const {
+	return _finished;
+}
+
+bool BinlogWrapper::failed() const {
+	return _failed;
+}
+
+std::optional<BasicHeader> BinlogWrapper::ReadHeader(
+		File &binlog,
+		const Settings &settings) {
+	auto result = BasicHeader();
+	if (binlog.offset() != 0) {
+		return {};
+	} else if (binlog.read(bytes::object_as_span(&result)) != sizeof(result)) {
+		return {};
+	} else if (result.getFormat() != Format::Format_0) {
+		return {};
+	} else if (settings.trackEstimatedTime
+		!= !!(result.flags & result.kTrackEstimatedTime)) {
+		return {};
+	}
+	return result;
+}
+
+bool BinlogWrapper::readPart() {
+	if (_finished) {
+		return false;
+	}
+	const auto no = [&] {
+		finish();
+		return false;
+	};
+	const auto offset = _binlog.offset();
+	const auto left = (_till - offset);
+	if (!left) {
+		return no();
+	}
+
+	if (!_part.empty() && _full.data() != _part.data()) {
+		bytes::move(_full, _part);
+		_part = _full.subspan(0, _part.size());
+	}
+	const auto amount = std::min(
+		left,
+		int64(_full.size() - _part.size()));
+	Assert(amount > 0);
+	const auto readBytes = _binlog.read(
+		_full.subspan(_part.size(), amount));
+	if (!readBytes) {
+		return no();
+	}
+	_part = _full.subspan(0, _part.size() + readBytes);
+	return true;
+}
+
+bytes::const_span BinlogWrapper::readRecord(ReadRecordSize readRecordSize) {
+	if (_finished) {
+		return {};
+	}
+	const auto size = readRecordSize(*this, _part);
+	if (size == kRecordSizeUnknown || size > _part.size()) {
+		return {};
+	} else if (size == kRecordSizeInvalid) {
+		finish();
+		_finished = _failed = true;
+		return {};
+	}
+	Assert(size >= 0);
+	const auto result = _part.subspan(0, size);
+	_part = _part.subspan(size);
+	return result;
+}
+
+void BinlogWrapper::finish(size_type rollback) {
+	Expects(rollback >= 0);
+
+	if (rollback > 0) {
+		_failed = true;
+	}
+	rollback += _part.size();
+	_binlog.seek(_binlog.offset() - rollback);
+}
+
+} // namespace details
+} // namespace Cache
+} // namespace Storage

Telegram/lib_storage/storage/cache/storage_cache_binlog_reader.h

@@ -0,0 +1,285 @@
+// This file is part of Desktop App Toolkit,
+// a set of libraries for developing nice desktop applications.
+//
+// For license and copyright information please follow this link:
+// https://github.com/desktop-app/legal/blob/master/LEGAL
+//
+#pragma once
+
+#include "storage/cache/storage_cache_types.h"
+#include "storage/storage_encrypted_file.h"
+#include "base/bytes.h"
+#include "base/match_method.h"
+
+namespace Storage {
+namespace Cache {
+namespace details {
+
+template <typename ...Records>
+class BinlogReader;
+
+class BinlogWrapper {
+public:
+	BinlogWrapper(File &binlog, const Settings &settings, int64 till = 0);
+
+	bool finished() const;
+	bool failed() const;
+
+	static std::optional<BasicHeader> ReadHeader(
+		File &binlog,
+		const Settings &settings);
+
+private:
+	template <typename ...Records>
+	friend class BinlogReader;
+
+	bool readPart();
+	void finish(size_type rollback = 0);
+
+	using ReadRecordSize = size_type (*)(
+		const BinlogWrapper &that,
+		bytes::const_span data);
+	bytes::const_span readRecord(ReadRecordSize readRecordSize);
+
+	File &_binlog;
+	Settings _settings;
+
+	int64 _till = 0;
+	bytes::vector _data;
+	bytes::span _full;
+	bytes::span _part;
+	bool _finished = false;
+	bool _failed = false;
+
+};
+
+template <typename ...Records>
+class BinlogReader {
+public:
+	explicit BinlogReader(BinlogWrapper &wrapper);
+
+	template <typename ...Handlers>
+	bool readTillEnd(Handlers &&...handlers);
+
+private:
+	static size_type ReadRecordSize(
+		const BinlogWrapper &that,
+		bytes::const_span data);
+
+	template <typename ...Handlers>
+	bool handleRecord(bytes::const_span data, Handlers &&...handlers) const;
+
+	BinlogWrapper &_wrapper;
+
+};
+
+template <typename Record>
+struct MultiRecord {
+	using true_t = char;
+	using false_t = true_t(&)[2];
+	static_assert(sizeof(true_t) != sizeof(false_t));
+
+	static false_t Check(...);
+	template <typename Test, typename = typename Test::Part>
+	static true_t Check(const Test&);
+
+	static constexpr bool Is = (sizeof(Check(std::declval<Record>()))
+		== sizeof(true_t));
+};
+
+template <typename ...Records>
+struct BinlogReaderRecursive {
+	static void CheckSettings(const Settings &settings) {
+	}
+	static void CheckSettingsMulti(const Settings &settings) {
+	}
+
+	static size_type ReadRecordSize(
+			RecordType type,
+			bytes::const_span data,
+			size_type partsLimit) {
+		return kRecordSizeInvalid;
+	}
+
+	template <typename ...Handlers>
+	static bool HandleRecord(
+			RecordType type,
+			bytes::const_span data,
+			Handlers &&...handlers) {
+		Unexpected("Bad type in BinlogReaderRecursive::HandleRecord.");
+	}
+};
+
+template <typename Record, typename ...Other>
+struct BinlogReaderRecursive<Record, Other...> {
+	static void CheckSettings(const Settings &settings);
+	static void CheckSettingsMulti(const Settings &settings);
+
+	static size_type ReadRecordSize(
+		RecordType type,
+		bytes::const_span data,
+		size_type partsLimit);
+	static size_type ReadRecordSizeMulti(
+		RecordType type,
+		bytes::const_span data,
+		size_type partsLimit);
+
+	template <typename ...Handlers>
+	static bool HandleRecord(
+		RecordType type,
+		bytes::const_span data,
+		Handlers &&...handlers);
+};
+
+template <typename Record, typename ...Other>
+inline void BinlogReaderRecursive<Record, Other...>::CheckSettings(
+		const Settings &settings) {
+	static_assert(GoodForEncryption<Record>);
+	if constexpr (MultiRecord<Record>::Is) {
+		CheckSettingsMulti(settings);
+	} else {
+		Assert(settings.readBlockSize >= sizeof(Record));
+	}
+}
+
+template <typename Record, typename ...Other>
+inline void BinlogReaderRecursive<Record, Other...>::CheckSettingsMulti(
+		const Settings &settings) {
+	using Head = Record;
+	using Part = typename Record::Part;
+	static_assert(GoodForEncryption<Part>);
+	Assert(settings.readBlockSize
+		>= (sizeof(Head)
+			+ settings.maxBundledRecords * sizeof(Part)));
+}
+
+template <typename Record, typename ...Other>
+inline size_type BinlogReaderRecursive<Record, Other...>::ReadRecordSize(
+		RecordType type,
+		bytes::const_span data,
+		size_type partsLimit) {
+	if (type != Record::kType) {
+		return BinlogReaderRecursive<Other...>::ReadRecordSize(
+			type,
+			data,
+			partsLimit);
+	}
+	if constexpr (MultiRecord<Record>::Is) {
+		return ReadRecordSizeMulti(type, data, partsLimit);
+	} else {
+		return sizeof(Record);
+	}
+}
+
+template <typename Record, typename ...Other>
+inline size_type BinlogReaderRecursive<Record, Other...>::ReadRecordSizeMulti(
+		RecordType type,
+		bytes::const_span data,
+		size_type partsLimit) {
+	using Head = Record;
+	using Part = typename Record::Part;
+
+	if (data.size() < sizeof(Head)) {
+		return kRecordSizeUnknown;
+	}
+	const auto head = reinterpret_cast<const Head*>(data.data());
+	const auto count = head->validateCount();
+	return (count >= 0 && count <= partsLimit)
+		? (sizeof(Head) + count * sizeof(Part))
+		: kRecordSizeInvalid;
+}
+
+template <typename Record, typename ...Other>
+template <typename ...Handlers>
+inline bool BinlogReaderRecursive<Record, Other...>::HandleRecord(
+		RecordType type,
+		bytes::const_span data,
+		Handlers &&...handlers) {
+	if (type != Record::kType) {
+		return BinlogReaderRecursive<Other...>::HandleRecord(
+			type,
+			data,
+			std::forward<Handlers>(handlers)...);
+	}
+	if constexpr (MultiRecord<Record>::Is) {
+		using Head = Record;
+		using Part = typename Record::Part;
+
+		Assert(data.size() >= sizeof(Head));
+		const auto bytes = data.data();
+		const auto head = reinterpret_cast<const Head*>(bytes);
+		const auto count = head->validateCount();
+		Assert(data.size() == sizeof(Head) + count * sizeof(Part));
+		const auto parts = gsl::make_span(
+			reinterpret_cast<const Part*>(bytes + sizeof(Head)),
+			count);
+		auto from = std::begin(parts);
+		const auto till = std::end(parts);
+		const auto element = [&] {
+			return (from == till) ? nullptr : &*from++;
+		};
+		return base::match_method2(
+			*head,
+			element,
+			std::forward<Handlers>(handlers)...);
+	} else {
+		Assert(data.size() == sizeof(Record));
+		return base::match_method(
+			*reinterpret_cast<const Record*>(data.data()),
+			std::forward<Handlers>(handlers)...);
+	}
+}
+
+template <typename ...Records>
+BinlogReader<Records...>::BinlogReader(BinlogWrapper &wrapper)
+: _wrapper(wrapper) {
+	BinlogReaderRecursive<Records...>::CheckSettings(_wrapper._settings);
+}
+
+template <typename ...Records>
+template <typename ...Handlers>
+bool BinlogReader<Records...>::readTillEnd(Handlers &&...handlers) {
+	if (!_wrapper.readPart()) {
+		return true;
+	}
+	const auto readRecord = [&] {
+		return _wrapper.readRecord(&BinlogReader::ReadRecordSize);
+	};
+	for (auto bytes = readRecord(); !bytes.empty(); bytes = readRecord()) {
+		if (!handleRecord(bytes, std::forward<Handlers>(handlers)...)) {
+			_wrapper.finish(bytes.size());
+			return true;
+		}
+	}
+	return false;
+}
+
+template <typename ...Records>
+size_type BinlogReader<Records...>::ReadRecordSize(
+		const BinlogWrapper &that,
+		bytes::const_span data) {
+	if (data.empty()) {
+		return kRecordSizeUnknown;
+	}
+	return BinlogReaderRecursive<Records...>::ReadRecordSize(
+		static_cast<RecordType>(data[0]),
+		data,
+		that._settings.maxBundledRecords);
+}
+
+template <typename ...Records>
+template <typename ...Handlers>
+bool BinlogReader<Records...>::handleRecord(
+		bytes::const_span data,
+		Handlers &&...handlers) const {
+	Expects(!data.empty());
+
+	return BinlogReaderRecursive<Records...>::HandleRecord(
+		static_cast<RecordType>(data[0]),
+		data,
+		std::forward<Handlers>(handlers)...);
+}
+
+} // namespace details
+} // namespace Cache
+} // namespace Storage

Telegram/lib_storage/storage/cache/storage_cache_cleaner.cpp

@@ -0,0 +1,110 @@
+// This file is part of Desktop App Toolkit,
+// a set of libraries for developing nice desktop applications.
+//
+// For license and copyright information please follow this link:
+// https://github.com/desktop-app/legal/blob/master/LEGAL
+//
+#include "storage/cache/storage_cache_cleaner.h"
+
+#include <crl/crl.h>
+#include <QtCore/QDir>
+#include <QtCore/QFile>
+#include <unordered_map>
+#include <set>
+
+namespace Storage {
+namespace Cache {
+namespace details {
+
+class CleanerObject {
+public:
+	CleanerObject(
+		crl::weak_on_queue<CleanerObject> weak,
+		const QString &base,
+		base::binary_guard &&guard,
+		FnMut<void(Error)> done);
+
+private:
+	void start();
+	void scheduleNext();
+	void cleanNext();
+	void done();
+
+	crl::weak_on_queue<CleanerObject> _weak;
+	QString _base, _errorPath;
+	std::vector<QString> _queue;
+	base::binary_guard _guard;
+	FnMut<void(Error)> _done;
+
+};
+
+CleanerObject::CleanerObject(
+	crl::weak_on_queue<CleanerObject> weak,
+	const QString &base,
+	base::binary_guard &&guard,
+	FnMut<void(Error)> done)
+: _weak(std::move(weak))
+, _base(base)
+, _guard(std::move(guard))
+, _done(std::move(done)) {
+	start();
+}
+
+void CleanerObject::start() {
+	const auto entries = QDir(_base).entryList(
+		QDir::Dirs | QDir::NoDotAndDotDot);
+	for (const auto &entry : entries) {
+		_queue.push_back(entry);
+	}
+	if (const auto version = ReadVersionValue(_base)) {
+		_queue.erase(
+			ranges::remove(_queue, QString::number(*version)),
+			end(_queue));
+		scheduleNext();
+	} else {
+		_errorPath = VersionFilePath(_base);
+		done();
+	}
+}
+
+void CleanerObject::scheduleNext() {
+	if (_queue.empty()) {
+		done();
+		return;
+	}
+	_weak.with([](CleanerObject &that) {
+		if (that._guard) {
+			that.cleanNext();
+		}
+	});
+}
+
+void CleanerObject::cleanNext() {
+	const auto path = _base + _queue.back();
+	_queue.pop_back();
+	if (!QDir(path).removeRecursively()) {
+		_errorPath = path;
+	}
+	scheduleNext();
+}
+
+void CleanerObject::done() {
+	if (_done) {
+		_done(_errorPath.isEmpty()
+			? Error::NoError()
+			: Error{ Error::Type::IO, _errorPath });
+	}
+}
+
+Cleaner::Cleaner(
+	const QString &base,
+	base::binary_guard &&guard,
+	FnMut<void(Error)> done)
+: _wrapped(base, std::move(guard), std::move(done)) {
+}
+
+Cleaner::~Cleaner() = default;
+
+} // namespace details
+} // namespace Cache
+} // namespace Storage

Telegram/lib_storage/storage/cache/storage_cache_cleaner.h

@@ -0,0 +1,35 @@
+// This file is part of Desktop App Toolkit,
+// a set of libraries for developing nice desktop applications.
+//
+// For license and copyright information please follow this link:
+// https://github.com/desktop-app/legal/blob/master/LEGAL
+//
+#pragma once
+
+#include "storage/cache/storage_cache_types.h"
+#include "base/binary_guard.h"
+
+namespace Storage {
+namespace Cache {
+namespace details {
+
+class CleanerObject;
+
+class Cleaner {
+public:
+	Cleaner(
+		const QString &base,
+		base::binary_guard &&guard,
+		FnMut<void(Error)> done);
+
+	~Cleaner();
+
+private:
+	using Implementation = details::CleanerObject;
+	crl::object_on_queue<Implementation> _wrapped;
+
+};
+
+} // namespace details
+} // namespace Cache
+} // namespace Storage

Telegram/lib_storage/storage/cache/storage_cache_compactor.cpp

@@ -0,0 +1,430 @@
+// This file is part of Desktop App Toolkit,
+// a set of libraries for developing nice desktop applications.
+//
+// For license and copyright information please follow this link:
+// https://github.com/desktop-app/legal/blob/master/LEGAL
+//
+#include "storage/cache/storage_cache_compactor.h"
+
+#include "storage/cache/storage_cache_database_object.h"
+#include "storage/cache/storage_cache_binlog_reader.h"
+#include <unordered_set>
+
+namespace Storage {
+namespace Cache {
+namespace details {
+
+class CompactorObject {
+public:
+	using Info = Compactor::Info;
+
+	CompactorObject(
+		crl::weak_on_queue<CompactorObject> weak,
+		crl::weak_on_queue<DatabaseObject> database,
+		base::binary_guard guard,
+		const QString &base,
+		const Settings &settings,
+		EncryptionKey &&key,
+		const Info &info);
+
+private:
+	using Entry = DatabaseObject::Entry;
+	using Raw = DatabaseObject::Raw;
+	using RawSpan = gsl::span<const Raw>;
+	static QString CompactFilename();
+
+	void start();
+	QString binlogPath() const;
+	QString compactPath() const;
+	bool openBinlog();
+	bool readHeader();
+	bool openCompact();
+	void parseChunk();
+	void fail();
+	void done(int64 till);
+	void finish();
+	void finalize();
+
+	std::vector<Key> readChunk();
+	bool readBlock(std::vector<Key> &result);
+	void processValues(const std::vector<Raw> &values);
+
+	template <typename MultiRecord>
+	void initList();
+	RawSpan fillList(RawSpan values);
+	template <typename RecordStore>
+	RawSpan fillList(std::vector<RecordStore> &list, RawSpan values);
+	template <typename RecordStore>
+	void addListRecord(
+		std::vector<RecordStore> &list,
+		const Raw &raw);
+	bool writeList();
+	template <typename MultiRecord>
+	bool writeMultiStore();
+
+	crl::weak_on_queue<CompactorObject> _weak;
+	crl::weak_on_queue<DatabaseObject> _database;
+	base::binary_guard _guard;
+	QString _base;
+	Settings _settings;
+	EncryptionKey _key;
+	BasicHeader _header;
+	Info _info;
+	File _binlog;
+	File _compact;
+	BinlogWrapper _wrapper;
+	size_type _partSize = 0;
+	std::unordered_set<Key> _written;
+	std::variant<
+		std::vector<MultiStore::Part>,
+		std::vector<MultiStoreWithTime::Part>> _list;
+
+};
+
+CompactorObject::CompactorObject(
+	crl::weak_on_queue<CompactorObject> weak,
+	crl::weak_on_queue<DatabaseObject> database,
+	base::binary_guard guard,
+	const QString &base,
+	const Settings &settings,
+	EncryptionKey &&key,
+	const Info &info)
+: _weak(std::move(weak))
+, _database(std::move(database))
+, _guard(std::move(guard))
+, _base(base)
+, _settings(settings)
+, _key(std::move(key))
+, _info(info)
+, _wrapper(_binlog, _settings, _info.till)
+, _partSize(_settings.maxBundledRecords) { // Perhaps a better estimate?
+	Expects(_settings.compactChunkSize > 0);
+
+	_written.reserve(_info.keysCount);
+	start();
+}
+
+template <typename MultiRecord>
+void CompactorObject::initList() {
+	using Part = typename MultiRecord::Part;
+	auto list = std::vector<Part>();
+	list.reserve(_partSize);
+	_list = std::move(list);
+}
+
+void CompactorObject::start() {
+	if (!openBinlog() || !readHeader() || !openCompact()) {
+		fail();
+	}
+	if (_settings.trackEstimatedTime) {
+		initList<MultiStoreWithTime>();
+	} else {
+		initList<MultiStore>();
+	}
+	parseChunk();
+}
+
+QString CompactorObject::CompactFilename() {
+	return QStringLiteral("binlog-temp");
+}
+
+QString CompactorObject::binlogPath() const {
+	return _base + DatabaseObject::BinlogFilename();
+}
+
+QString CompactorObject::compactPath() const {
+	return _base + CompactFilename();
+}
+
+bool CompactorObject::openBinlog() {
+	const auto path = binlogPath();
+	const auto result = _binlog.open(path, File::Mode::Read, _key);
+	return (result == File::Result::Success)
+		&& (_binlog.size() >= _info.till);
+}
+
+bool CompactorObject::readHeader() {
+	const auto header = BinlogWrapper::ReadHeader(_binlog, _settings);
+	if (!header) {
+		return false;
+	}
+	_header = *header;
+	return true;
+}
+
+bool CompactorObject::openCompact() {
+	const auto path = compactPath();
+	const auto result = _compact.open(path, File::Mode::Write, _key);
+	if (result != File::Result::Success) {
+		return false;
+	} else if (!_compact.write(bytes::object_as_span(&_header))) {
+		return false;
+	}
+	return true;
+}
+
+void CompactorObject::fail() {
+	_compact.close();
+	QFile(compactPath()).remove();
+	_database.with([](DatabaseObject &database) {
+		database.compactorFail();
+	});
+}
+
+void CompactorObject::done(int64 till) {
+	const auto path = compactPath();
+	_database.with([=, good = std::move(_guard)](DatabaseObject &database) {
+		if (good) {
+			database.compactorDone(path, till);
+		}
+	});
+}
+
+void CompactorObject::finish() {
+	if (writeList()) {
+		finalize();
+	} else {
+		fail();
+	}
+}
+
+void CompactorObject::finalize() {
+	_binlog.close();
+	_compact.close();
+
+	auto lastCatchUp = 0;
+	auto from = _info.till;
+	while (true) {
+		const auto till = CatchUp(
+			compactPath(),
+			binlogPath(),
+			_key,
+			from,
+			_settings.readBlockSize);
+		if (!till) {
+			fail();
+			return;
+		} else if (till == from
+			|| (lastCatchUp > 0 && (till - from) >= lastCatchUp)) {
+			done(till);
+			return;
+		}
+		lastCatchUp = (till - from);
+		from = till;
+	}
+}
+
+bool CompactorObject::writeList() {
+	return v::match(_list, [&](const std::vector<MultiStore::Part>&) {
+		return writeMultiStore<MultiStore>();
+	}, [&](const std::vector<MultiStoreWithTime::Part>&) {
+		return writeMultiStore<MultiStoreWithTime>();
+	});
+}
+
+template <typename MultiRecord>
+bool CompactorObject::writeMultiStore() {
+	using Part = typename MultiRecord::Part;
+	auto &list = v::get<std::vector<Part>>(_list);
+	if (list.empty()) {
+		return true;
+	}
+	const auto guard = gsl::finally([&] { list.clear(); });
+	const auto size = list.size();
+	auto header = MultiRecord(size);
+	if (_compact.write(bytes::object_as_span(&header))
+		&& _compact.write(bytes::make_span(list))) {
+		_compact.flush();
+		return true;
+	}
+	return false;
+}
+
+std::vector<Key> CompactorObject::readChunk() {
+	const auto limit = _settings.compactChunkSize;
+	auto result = std::vector<Key>();
+	while (result.size() < limit) {
+		if (!readBlock(result)) {
+			break;
+		}
+	}
+	return result;
+}
+
+bool CompactorObject::readBlock(std::vector<Key> &result) {
+	const auto push = [&](const Store &store) {
+		result.push_back(store.key);
+		return true;
+	};
+	const auto pushMulti = [&](const auto &element) {
+		while (const auto record = element()) {
+			push(*record);
+		}
+		return true;
+	};
+	if (_settings.trackEstimatedTime) {
+		BinlogReader<
+			StoreWithTime,
+			MultiStoreWithTime,
+			MultiRemove,
+			MultiAccess> reader(_wrapper);
+		return !reader.readTillEnd([&](const StoreWithTime &record) {
+			return push(record);
+		}, [&](const MultiStoreWithTime &header, const auto &element) {
+			return pushMulti(element);
+		}, [&](const MultiRemove &header, const auto &element) {
+			return true;
+		}, [&](const MultiAccess &header, const auto &element) {
+			return true;
+		});
+	} else {
+		BinlogReader<
+			Store,
+			MultiStore,
+			MultiRemove> reader(_wrapper);
+		return !reader.readTillEnd([&](const Store &record) {
+			return push(record);
+		}, [&](const MultiStore &header, const auto &element) {
+			return pushMulti(element);
+		}, [&](const MultiRemove &header, const auto &element) {
+			return true;
+		});
+	}
+}
+
+void CompactorObject::parseChunk() {
+	auto keys = readChunk();
+	if (_wrapper.failed()) {
+		fail();
+		return;
+	} else if (keys.empty()) {
+		finish();
+		return;
+	}
+	_database.with([
+		weak = _weak,
+		keys = std::move(keys)
+	](DatabaseObject &database) {
+		auto result = database.getManyRaw(keys);
+		weak.with([result = std::move(result)](CompactorObject &that) {
+			that.processValues(result);
+		});
+	});
+}
+
+void CompactorObject::processValues(
+		const std::vector<std::pair<Key, Entry>> &values) {
+	auto left = gsl::make_span(values);
+	while (true) {
+		left = fillList(left);
+		if (left.empty()) {
+			break;
+		} else if (!writeList()) {
+			fail();
+			return;
+		}
+	}
+	parseChunk();
+}
+
+auto CompactorObject::fillList(RawSpan values) -> RawSpan {
+	return v::match(_list, [&](auto &list) {
+		return fillList(list, values);
+	});
+}
+
+template <typename RecordStore>
+auto CompactorObject::fillList(
+	std::vector<RecordStore> &list,
+	RawSpan values
+) -> RawSpan {
+	const auto b = std::begin(values);
+	const auto e = std::end(values);
+	auto i = b;
+	while (i != e && list.size() != _partSize) {
+		addListRecord(list, *i++);
+	}
+	return values.subspan(i - b);
+}
+
+template <typename RecordStore>
+void CompactorObject::addListRecord(
+		std::vector<RecordStore> &list,
+		const Raw &raw) {
+	if (!_written.emplace(raw.first).second) {
+		return;
+	}
+	auto record = RecordStore();
+	record.key = raw.first;
+	record.setSize(raw.second.size);
+	record.checksum = raw.second.checksum;
+	record.tag = raw.second.tag;
+	record.place = raw.second.place;
+	if constexpr (std::is_same_v<RecordStore, StoreWithTime>) {
+		record.time.setRelative(raw.second.useTime);
+		record.time.system = _info.systemTime;
+	}
+	list.push_back(record);
+}
+
+Compactor::Compactor(
+	crl::weak_on_queue<DatabaseObject> database,
+	base::binary_guard guard,
+	const QString &base,
+	const Settings &settings,
+	EncryptionKey &&key,
+	const Info &info)
+: _wrapped(
+	std::move(database),
+	std::move(guard),
+	base,
+	settings,
+	std::move(key),
+	info) {
+}
+
+Compactor::~Compactor() = default;
+
+int64 CatchUp(
+		const QString &compactPath,
+		const QString &binlogPath,
+		const EncryptionKey &key,
+		int64 from,
+		size_type block) {
+	File binlog, compact;
+	const auto result1 = binlog.open(binlogPath, File::Mode::Read, key);
+	if (result1 != File::Result::Success) {
+		return 0;
+	}
+	const auto till = binlog.size();
+	if (till == from) {
+		return till;
+	} else if (till < from || !binlog.seek(from)) {
+		return 0;
+	}
+	const auto result2 = compact.open(
+		compactPath,
+		File::Mode::ReadAppend,
+		key);
+	if (result2 != File::Result::Success || !compact.seek(compact.size())) {
+		return 0;
+	}
+	auto buffer = bytes::vector(block);
+	auto bytes = bytes::make_span(buffer);
+	do {
+		const auto left = (till - from);
+		const auto limit = std::min(size_type(left), block);
+		const auto read = binlog.read(bytes.subspan(0, limit));
+		if (!read || read > limit) {
+			return 0;
+		} else if (!compact.write(bytes.subspan(0, read))) {
+			return 0;
+		}
+		from += read;
+	} while (from != till);
+	return till;
+}
+
+} // namespace details
+} // namespace Cache
+} // namespace Storage

Telegram/lib_storage/storage/cache/storage_cache_compactor.h

@@ -0,0 +1,54 @@
+// This file is part of Desktop App Toolkit,
+// a set of libraries for developing nice desktop applications.
+//
+// For license and copyright information please follow this link:
+// https://github.com/desktop-app/legal/blob/master/LEGAL
+//
+#pragma once
+
+#include "storage/cache/storage_cache_types.h"
+#include <crl/crl_object_on_queue.h>
+#include <base/binary_guard.h>
+
+namespace Storage {
+class EncryptionKey;
+namespace Cache {
+namespace details {
+
+class CompactorObject;
+class DatabaseObject;
+
+class Compactor {
+public:
+	struct Info {
+		int64 till = 0;
+		uint32 systemTime = 0;
+		size_type keysCount = 0;
+	};
+
+	Compactor(
+		crl::weak_on_queue<DatabaseObject> database,
+		base::binary_guard guard,
+		const QString &base,
+		const Settings &settings,
+		EncryptionKey &&key,
+		const Info &info);
+
+	~Compactor();
+
+private:
+	using Implementation = CompactorObject;
+	crl::object_on_queue<Implementation> _wrapped;
+
+};
+
+int64 CatchUp(
+	const QString &compactPath,
+	const QString &binlogPath,
+	const EncryptionKey &key,
+	int64 from,
+	size_type block);
+
+} // namespace details
+} // namespace Cache
+} // namespace Storage

Telegram/lib_storage/storage/cache/storage_cache_database.cpp

@@ -0,0 +1,202 @@
+// This file is part of Desktop App Toolkit,
+// a set of libraries for developing nice desktop applications.
+//
+// For license and copyright information please follow this link:
+// https://github.com/desktop-app/legal/blob/master/LEGAL
+//
+#include "storage/cache/storage_cache_database.h"
+
+#include "storage/cache/storage_cache_database_object.h"
+
+namespace Storage {
+namespace Cache {
+
+Database::Database(const QString &path, const Settings &settings)
+: _wrapped(path, settings) {
+}
+
+void Database::reconfigure(const Settings &settings) {
+	_wrapped.with([settings](Implementation &unwrapped) mutable {
+		unwrapped.reconfigure(settings);
+	});
+}
+
+void Database::updateSettings(const SettingsUpdate &update) {
+	_wrapped.with([update](Implementation &unwrapped) mutable {
+		unwrapped.updateSettings(update);
+	});
+}
+
+void Database::open(EncryptionKey &&key, FnMut<void(Error)> &&done) {
+	_wrapped.with([
+		key = std::move(key),
+		done = std::move(done)
+	](Implementation &unwrapped) mutable {
+		unwrapped.open(std::move(key), std::move(done));
+	});
+}
+
+void Database::close(FnMut<void()> &&done) {
+	_wrapped.with([
+		done = std::move(done)
+	](Implementation &unwrapped) mutable {
+		unwrapped.close(std::move(done));
+	});
+}
+
+void Database::waitForCleaner(FnMut<void()> &&done) {
+	_wrapped.with([
+		done = std::move(done)
+	](Implementation &unwrapped) mutable {
+		unwrapped.waitForCleaner(std::move(done));
+	});
+}
+
+void Database::put(
+		const Key &key,
+		QByteArray &&value,
+		FnMut<void(Error)> &&done) {
+	return put(key, TaggedValue(std::move(value), 0), std::move(done));
+}
+
+void Database::get(const Key &key, FnMut<void(QByteArray&&)> &&done) {
+	if (done) {
+		auto untag = [done = std::move(done)](TaggedValue &&value) mutable {
+			done(std::move(value.bytes));
+		};
+		getWithTag(key, std::move(untag));
+	} else {
+		getWithTag(key, nullptr);
+	}
+}
+
+void Database::remove(const Key &key, FnMut<void(Error)> &&done) {
+	_wrapped.with([
+		key,
+		done = std::move(done)
+	](Implementation &unwrapped) mutable {
+		unwrapped.remove(key, std::move(done));
+	});
+}
+
+void Database::putIfEmpty(
+		const Key &key,
+		QByteArray &&value,
+		FnMut<void(Error)> &&done) {
+	return putIfEmpty(
+		key,
+		TaggedValue(std::move(value), 0),
+		std::move(done));
+}
+
+void Database::copyIfEmpty(
+		const Key &from,
+		const Key &to,
+		FnMut<void(Error)> &&done) {
+	_wrapped.with([
+		from,
+		to,
+		done = std::move(done)
+	](Implementation &unwrapped) mutable {
+		unwrapped.copyIfEmpty(from, to, std::move(done));
+	});
+}
+
+void Database::moveIfEmpty(
+		const Key &from,
+		const Key &to,
+		FnMut<void(Error)> &&done) {
+	_wrapped.with([
+		from,
+		to,
+		done = std::move(done)
+	](Implementation &unwrapped) mutable {
+		unwrapped.moveIfEmpty(from, to, std::move(done));
+	});
+}
+
+void Database::put(
+		const Key &key,
+		TaggedValue &&value,
+		FnMut<void(Error)> &&done) {
+	_wrapped.with([
+		key,
+		value = std::move(value),
+		done = std::move(done)
+	](Implementation &unwrapped) mutable {
+		unwrapped.put(key, std::move(value), std::move(done));
+	});
+}
+
+void Database::putIfEmpty(
+		const Key &key,
+		TaggedValue &&value,
+		FnMut<void(Error)> &&done) {
+	_wrapped.with([
+		key,
+		value = std::move(value),
+		done = std::move(done)
+	](Implementation &unwrapped) mutable {
+		unwrapped.putIfEmpty(key, std::move(value), std::move(done));
+	});
+}
+
+void Database::getWithTag(
+		const Key &key,
+		FnMut<void(TaggedValue&&)> &&done) {
+	_wrapped.with([
+		key,
+		done = std::move(done)
+	](Implementation &unwrapped) mutable {
+		unwrapped.get(key, std::move(done));
+	});
+}
+
+void Database::getWithSizes(
+		const Key &key,
+		std::vector<Key> &&keys,
+		FnMut<void(QByteArray&&, std::vector<int>&&)> &&done) {
+	_wrapped.with([
+		key,
+		keys = std::move(keys),
+		done = std::move(done)
+	](Implementation &unwrapped) mutable {
+		unwrapped.getWithSizes(key, std::move(keys), std::move(done));
+	});
+}
+
+auto Database::statsOnMain() const -> rpl::producer<Stats> {
+	return _wrapped.producer_on_main([](const Implementation &unwrapped) {
+		return unwrapped.stats();
+	});
+}
+
+void Database::clear(FnMut<void(Error)> &&done) {
+	_wrapped.with([
+		done = std::move(done)
+	](Implementation &unwrapped) mutable {
+		unwrapped.clear(std::move(done));
+	});
+}
+
+void Database::clearByTag(uint8 tag, FnMut<void(Error)> &&done) {
+	_wrapped.with([
+		tag,
+		done = std::move(done)
+	](Implementation &unwrapped) mutable {
+		unwrapped.clearByTag(tag, std::move(done));
+	});
+}
+
+void Database::sync() {
+	auto semaphore = crl::semaphore();
+	_wrapped.with([&](Implementation &) {
+		semaphore.release();
+	});
+	semaphore.acquire();
+}
+
+Database::~Database() = default;
+
+} // namespace Cache
+} // namespace Storage

Telegram/lib_storage/storage/cache/storage_cache_database.h

@@ -0,0 +1,90 @@
+// This file is part of Desktop App Toolkit,
+// a set of libraries for developing nice desktop applications.
+//
+// For license and copyright information please follow this link:
+// https://github.com/desktop-app/legal/blob/master/LEGAL
+//
+#pragma once
+
+#include "storage/cache/storage_cache_types.h"
+#include "base/basic_types.h"
+#include <crl/crl_object_on_queue.h>
+#include <crl/crl_time.h>
+#include <rpl/producer.h>
+#include <QtCore/QString>
+
+namespace Storage {
+class EncryptionKey;
+namespace Cache {
+namespace details {
+class DatabaseObject;
+} // namespace details
+
+class Database {
+public:
+	using Settings = details::Settings;
+	using SettingsUpdate = details::SettingsUpdate;
+	Database(const QString &path, const Settings &settings);
+
+	void reconfigure(const Settings &settings);
+	void updateSettings(const SettingsUpdate &update);
+
+	void open(EncryptionKey &&key, FnMut<void(Error)> &&done = nullptr);
+	void close(FnMut<void()> &&done = nullptr);
+
+	void put(
+		const Key &key,
+		QByteArray &&value,
+		FnMut<void(Error)> &&done = nullptr);
+	void get(const Key &key, FnMut<void(QByteArray&&)> &&done);
+	void remove(const Key &key, FnMut<void(Error)> &&done = nullptr);
+
+	void putIfEmpty(
+		const Key &key,
+		QByteArray &&value,
+		FnMut<void(Error)> &&done = nullptr);
+	void copyIfEmpty(
+		const Key &from,
+		const Key &to,
+		FnMut<void(Error)> &&done = nullptr);
+	void moveIfEmpty(
+		const Key &from,
+		const Key &to,
+		FnMut<void(Error)> &&done = nullptr);
+
+	using TaggedValue = details::TaggedValue;
+	void put(
+		const Key &key,
+		TaggedValue &&value,
+		FnMut<void(Error)> &&done = nullptr);
+	void putIfEmpty(
+		const Key &key,
+		TaggedValue &&value,
+		FnMut<void(Error)> &&done = nullptr);
+	void getWithTag(const Key &key, FnMut<void(TaggedValue&&)> &&done);
+
+	void getWithSizes(
+		const Key &key,
+		std::vector<Key> &&keys,
+		FnMut<void(QByteArray&&, std::vector<int>&&)> &&done);
+
+	using Stats = details::Stats;
+	using TaggedSummary = details::TaggedSummary;
+	rpl::producer<Stats> statsOnMain() const;
+
+	void clear(FnMut<void(Error)> &&done = nullptr);
+	void clearByTag(uint8 tag, FnMut<void(Error)> &&done = nullptr);
+	void waitForCleaner(FnMut<void()> &&done = nullptr);
+
+	void sync();
+
+	~Database();
+
+private:
+	using Implementation = details::DatabaseObject;
+	crl::object_on_queue<Implementation> _wrapped;
+
+};
+
+} // namespace Cache
+} // namespace Storage

Telegram/lib_storage/storage/cache/storage_cache_database_object.h

@@ -0,0 +1,269 @@
+// This file is part of Desktop App Toolkit,
+// a set of libraries for developing nice desktop applications.
+//
+// For license and copyright information please follow this link:
+// https://github.com/desktop-app/legal/blob/master/LEGAL
+//
+#pragma once
+
+#include "storage/cache/storage_cache_database.h"
+#include "storage/storage_encrypted_file.h"
+#include "base/binary_guard.h"
+#include "base/concurrent_timer.h"
+#include "base/bytes.h"
+#include "base/flat_set.h"
+#include <set>
+#include <rpl/event_stream.h>
+
+namespace Storage {
+namespace Cache {
+namespace details {
+
+class Cleaner;
+class Compactor;
+
+class DatabaseObject {
+public:
+	using Settings = Cache::Database::Settings;
+	DatabaseObject(
+		crl::weak_on_queue<DatabaseObject> weak,
+		const QString &path,
+		const Settings &settings);
+	void reconfigure(const Settings &settings);
+	void updateSettings(const SettingsUpdate &update);
+
+	void open(EncryptionKey &&key, FnMut<void(Error)> &&done);
+	void close(FnMut<void()> &&done);
+
+	void put(
+		const Key &key,
+		TaggedValue &&value,
+		FnMut<void(Error)> &&done);
+	void get(const Key &key, FnMut<void(TaggedValue&&)> &&done);
+	void remove(const Key &key, FnMut<void(Error)> &&done);
+
+	void putIfEmpty(
+		const Key &key,
+		TaggedValue &&value,
+		FnMut<void(Error)> &&done);
+	void copyIfEmpty(
+		const Key &from,
+		const Key &to,
+		FnMut<void(Error)> &&done);
+	void moveIfEmpty(
+		const Key &from,
+		const Key &to,
+		FnMut<void(Error)> &&done);
+
+	void getWithSizes(
+		const Key &key,
+		std::vector<Key> &&keys,
+		FnMut<void(QByteArray&&, std::vector<int>&&)> &&done);
+
+	rpl::producer<Stats> stats() const;
+
+	void clear(FnMut<void(Error)> &&done);
+	void clearByTag(uint8 tag, FnMut<void(Error)> &&done);
+	void waitForCleaner(FnMut<void()> &&done);
+
+	static QString BinlogFilename();
+	static QString CompactReadyFilename();
+
+	void compactorDone(const QString &path, int64 originalReadTill);
+	void compactorFail();
+
+	struct Entry {
+		Entry() = default;
+		Entry(
+			PlaceId place,
+			uint8 tag,
+			uint32 checksum,
+			size_type size,
+			uint64 useTime);
+
+		uint64 useTime = 0;
+		size_type size = 0;
+		uint32 checksum = 0;
+		PlaceId place = { { 0 } };
+		uint8 tag = 0;
+	};
+	using Raw = std::pair<Key, Entry>;
+	std::vector<Raw> getManyRaw(const std::vector<Key> &keys) const;
+
+	~DatabaseObject();
+
+private:
+	struct CleanerWrap {
+		std::unique_ptr<Cleaner> object;
+		base::binary_guard guard;
+		FnMut<void()> done;
+	};
+	struct CompactorWrap {
+		std::unique_ptr<Compactor> object;
+		int64 excessLength = 0;
+		crl::time nextAttempt = 0;
+		crl::time delayAfterFailure = 10 * crl::time(1000);
+		base::binary_guard guard;
+	};
+	struct KeyPlaceChange {
+		QString wasPath;
+		QString nowPath;
+		PlaceId nowPlace;
+	};
+	using Map = std::unordered_map<Key, Entry>;
+
+	template <typename Callback, typename ...Args>
+	void invokeCallback(Callback &&callback, Args &&...args) const;
+
+	Error ioError(const QString &path) const;
+
+	void checkSettings();
+	QString computePath(Version version) const;
+	QString binlogPath(Version version) const;
+	QString binlogPath() const;
+	QString compactReadyPath(Version version) const;
+	QString compactReadyPath() const;
+	Error openSomeBinlog(EncryptionKey &&key);
+	Error openNewBinlog(EncryptionKey &key);
+	File::Result openBinlog(
+		Version version,
+		File::Mode mode,
+		EncryptionKey &key);
+	bool readHeader();
+	bool writeHeader();
+
+	void readBinlog();
+	template <typename Reader, typename ...Handlers>
+	void readBinlogHelper(Reader &reader, Handlers &&...handlers);
+	template <typename Record, typename Postprocess>
+	bool processRecordStoreGeneric(
+		const Record *record,
+		Postprocess &&postprocess);
+	bool processRecordStore(const Store *record, std::is_class<Store>);
+	bool processRecordStore(
+		const StoreWithTime *record,
+		std::is_class<StoreWithTime>);
+	template <typename Record, typename GetElement>
+	bool processRecordMultiStore(
+		const Record &header,
+		const GetElement &element);
+	template <typename GetElement>
+	bool processRecordMultiRemove(
+		const MultiRemove &header,
+		const GetElement &element);
+	template <typename GetElement>
+	bool processRecordMultiAccess(
+		const MultiAccess &header,
+		const GetElement &element);
+
+	void optimize();
+	void checkCompactor();
+	void adjustRelativeTime();
+	bool startDelayedPruning();
+	uint64 countRelativeTime() const;
+	EstimatedTimePoint countTimePoint() const;
+	void applyTimePoint(EstimatedTimePoint time);
+
+	uint64 pruneBeforeTime() const;
+	void prune();
+	void collectTimeStale(
+		base::flat_set<Key> &stale,
+		int64 &staleTotalSize);
+	void collectSizeStale(
+		base::flat_set<Key> &stale,
+		int64 &staleTotalSize);
+	void startStaleClear();
+	void clearStaleNow(const base::flat_set<Key> &stale);
+	void clearStaleChunkDelayed();
+	void clearStaleChunk();
+
+	void updateStats(const Entry &was, const Entry &now);
+	Stats collectStats() const;
+	void pushStatsDelayed();
+	void pushStats();
+
+	void setMapEntry(const Key &key, Entry &&entry);
+	void eraseMapEntry(const Map::const_iterator &i);
+	void recordEntryAccess(const Key &key);
+	QByteArray readValueData(PlaceId place, size_type size) const;
+
+	Version findAvailableVersion() const;
+	QString versionPath() const;
+	bool writeVersion(Version version);
+	Version readVersion() const;
+
+	QString placePath(PlaceId place) const;
+	bool isFreePlace(PlaceId place) const;
+
+	KeyPlaceChange chooseKeyPlace(
+		const Key &key,
+		const TaggedValue &value,
+		uint32 checksum);
+	Error writeNewEntry(
+		const QString &path,
+		QByteArray &&content);
+	template <typename StoreRecord>
+	Error writeKeyPlaceGeneric(
+		StoreRecord &&record,
+		const Key &key,
+		const PlaceId &place,
+		const TaggedValue &value,
+		uint32 checksum);
+	Error writeKeyPlace(
+		const Key &key,
+		const PlaceId &place,
+		const TaggedValue &value,
+		uint32 checksum);
+	template <typename StoreRecord>
+	Error writeExistingPlaceGeneric(
+		StoreRecord &&record,
+		const Key &key,
+		const Entry &entry);
+	Error writeExistingPlace(
+		const Key &key,
+		const Entry &entry);
+	void writeMultiRemoveLazy();
+	Error writeMultiRemove();
+	void writeMultiAccessLazy();
+	Error writeMultiAccess();
+	Error writeMultiAccessBlock();
+	void writeBundlesLazy();
+	void writeBundles();
+
+	void createCleaner();
+	void cleanerDone(Error error);
+	void clearState();
+
+	crl::weak_on_queue<DatabaseObject> _weak;
+	QString _base, _path;
+	Settings _settings;
+	EncryptionKey _key;
+	File _binlog;
+	Map _map;
+	std::set<Key> _removing;
+	std::set<Key> _accessed;
+	std::vector<Key> _stale;
+
+	EstimatedTimePoint _time;
+
+	int64 _binlogExcessLength = 0;
+	int64 _totalSize = 0;
+	uint64 _minimalEntryTime = 0;
+	size_type _entriesWithMinimalTimeCount = 0;
+
+	base::flat_map<uint8, TaggedSummary> _taggedStats;
+	rpl::event_stream<Stats> _stats;
+	bool _pushingStats = false;
+	bool _clearingStale = false;
+
+	base::ConcurrentTimer _writeBundlesTimer;
+	base::ConcurrentTimer _pruneTimer;
+
+	CleanerWrap _cleaner;
+	CompactorWrap _compactor;
+
+};
+
+} // namespace details
+} // namespace Cache
+} // namespace Storage

Telegram/lib_storage/storage/cache/storage_cache_types.cpp

@@ -0,0 +1,137 @@
+// This file is part of Desktop App Toolkit,
+// a set of libraries for developing nice desktop applications.
+//
+// For license and copyright information please follow this link:
+// https://github.com/desktop-app/legal/blob/master/LEGAL
+//
+#include "storage/cache/storage_cache_types.h"
+
+#include <QtCore/QDir>
+
+namespace Storage {
+namespace Cache {
+namespace details {
+namespace {
+
+template <typename Packed>
+inline Packed ReadTo(size_type count) {
+	Expects(count >= 0 && count < (size_type(1) << (Packed().size() * 8)));
+
+	auto result = Packed();
+	for (auto &element : result) {
+		element = uint8(count & 0xFF);
+		count >>= 8;
+	}
+	return result;
+}
+
+template <typename Packed>
+inline size_type ReadFrom(const Packed &count) {
+	auto result = size_type();
+	for (auto &element : (count | ranges::views::reverse)) {
+		result <<= 8;
+		result |= size_type(element);
+	}
+	return result;
+}
+
+template <typename Packed>
+inline size_type ValidateStrictCount(const Packed &count) {
+	const auto result = ReadFrom(count);
+	return (result != 0) ? result : -1;
+}
+
+} // namespace
+
+TaggedValue::TaggedValue(QByteArray &&bytes, uint8 tag)
+: bytes(std::move(bytes)), tag(tag) {
+}
+
+QString ComputeBasePath(const QString &original) {
+	const auto result = QDir(original).absolutePath();
+	return result.endsWith('/') ? result : (result + '/');
+}
+
+QString VersionFilePath(const QString &base) {
+	Expects(base.endsWith('/'));
+
+	return base + QStringLiteral("version");
+}
+
+std::optional<Version> ReadVersionValue(const QString &base) {
+	QFile file(VersionFilePath(base));
+	if (!file.open(QIODevice::ReadOnly)) {
+		return std::nullopt;
+	}
+	const auto bytes = file.read(sizeof(Version));
+	if (bytes.size() != sizeof(Version)) {
+		return std::nullopt;
+	}
+	return *reinterpret_cast<const Version*>(bytes.data());
+}
+
+bool WriteVersionValue(const QString &base, Version value) {
+	if (!QDir().mkpath(base)) {
+		return false;
+	}
+	const auto bytes = QByteArray::fromRawData(
+		reinterpret_cast<const char*>(&value),
+		sizeof(value));
+	QFile file(VersionFilePath(base));
+	if (!file.open(QIODevice::WriteOnly)) {
+		return false;
+	} else if (file.write(bytes) != bytes.size()) {
+		return false;
+	}
+	return file.flush();
+}
+
+BasicHeader::BasicHeader()
+: format(static_cast<uint32>(Format::Format_0))
+, flags(0) {
+}
+
+void Store::setSize(size_type size) {
+	this->size = ReadTo<EntrySize>(size);
+}
+
+size_type Store::getSize() const {
+	return ReadFrom(size);
+}
+
+MultiStore::MultiStore(size_type count)
+: type(kType)
+, count(ReadTo<RecordsCount>(count)) {
+	Expects(count >= 0 && count < kBundledRecordsLimit);
+}
+
+size_type MultiStore::validateCount() const {
+	return ValidateStrictCount(count);
+}
+
+MultiRemove::MultiRemove(size_type count)
+: type(kType)
+, count(ReadTo<RecordsCount>(count)) {
+	Expects(count >= 0 && count < kBundledRecordsLimit);
+}
+
+size_type MultiRemove::validateCount() const {
+	return ValidateStrictCount(count);
+}
+
+MultiAccess::MultiAccess(
+	EstimatedTimePoint time,
+	size_type count)
+: type(kType)
+, count(ReadTo<RecordsCount>(count))
+, time(time) {
+	Expects(count >= 0 && count < kBundledRecordsLimit);
+}
+
+size_type MultiAccess::validateCount() const {
+	return ReadFrom(count);
+}
+
+} // namespace details
+} // namespace Cache
+} // namespace Storage

Telegram/lib_storage/storage/cache/storage_cache_types.h

@@ -0,0 +1,245 @@
+// This file is part of Desktop App Toolkit,
+// a set of libraries for developing nice desktop applications.
+//
+// For license and copyright information please follow this link:
+// https://github.com/desktop-app/legal/blob/master/LEGAL
+//
+#pragma once
+
+#include "base/basic_types.h"
+#include "base/flat_map.h"
+#include "base/optional.h"
+#include <crl/crl_time.h>
+#include <QtCore/QString>
+#include <QtCore/QByteArray>
+
+namespace Storage {
+namespace Cache {
+
+struct Key {
+	uint64 high = 0;
+	uint64 low = 0;
+
+	[[nodiscard]] bool valid() const {
+		return (high != 0) || (low != 0);
+	}
+	explicit operator bool() const {
+		return valid();
+	}
+};
+
+inline bool operator==(const Key &a, const Key &b) {
+	return (a.high == b.high) && (a.low == b.low);
+}
+
+inline bool operator!=(const Key &a, const Key &b) {
+	return !(a == b);
+}
+
+inline bool operator<(const Key &a, const Key &b) {
+	return std::tie(a.high, a.low) < std::tie(b.high, b.low);
+}
+
+struct Error {
+	enum class Type {
+		None,
+		IO,
+		WrongKey,
+		LockFailed,
+	};
+	Type type = Type::None;
+	QString path;
+
+	static Error NoError();
+};
+
+inline Error Error::NoError() {
+	return Error();
+}
+
+namespace details {
+
+using RecordType = uint8;
+using PlaceId = std::array<uint8, 7>;
+using EntrySize = std::array<uint8, 3>;
+using RecordsCount = std::array<uint8, 3>;
+
+constexpr auto kRecordSizeUnknown = size_type(-1);
+constexpr auto kRecordSizeInvalid = size_type(-2);
+constexpr auto kBundledRecordsLimit
+	= size_type(1) << (RecordsCount().size() * 8);
+constexpr auto kDataSizeLimit = size_type(1) << (EntrySize().size() * 8);
+
+struct Settings {
+	size_type maxBundledRecords = 16 * 1024;
+	size_type readBlockSize = 8 * 1024 * 1024;
+	size_type maxDataSize = (kDataSizeLimit - 1);
+	crl::time writeBundleDelay = 15 * 60 * crl::time(1000);
+	size_type staleRemoveChunk = 256;
+
+	int64 compactAfterExcess = 8 * 1024 * 1024;
+	int64 compactAfterFullSize = 0;
+	size_type compactChunkSize = 16 * 1024;
+
+	bool trackEstimatedTime = true;
+	int64 totalSizeLimit = 1024 * 1024 * 1024;
+	size_type totalTimeLimit = 31 * 24 * 60 * 60; // One month in seconds.
+	crl::time pruneTimeout = 5 * crl::time(1000);
+	crl::time maxPruneCheckTimeout = 3600 * crl::time(1000);
+
+	bool clearOnWrongKey = false;
+};
+
+struct SettingsUpdate {
+	int64 totalSizeLimit = Settings().totalSizeLimit;
+	size_type totalTimeLimit = Settings().totalTimeLimit;
+};
+
+struct TaggedValue {
+	TaggedValue() = default;
+	TaggedValue(QByteArray &&bytes, uint8 tag);
+
+	QByteArray bytes;
+	uint8 tag = 0;
+};
+
+struct TaggedSummary {
+	size_type count = 0;
+	int64 totalSize = 0;
+};
+struct Stats {
+	TaggedSummary full;
+	base::flat_map<uint8, TaggedSummary> tagged;
+	bool clearing = false;
+};
+
+using Version = int32;
+
+QString ComputeBasePath(const QString &original);
+QString VersionFilePath(const QString &base);
+std::optional<Version> ReadVersionValue(const QString &base);
+bool WriteVersionValue(const QString &base, Version value);
+
+template <typename Record>
+constexpr auto GoodForEncryption = ((sizeof(Record) & 0x0F) == 0);
+
+enum class Format : uint32 {
+	Format_0,
+};
+
+struct BasicHeader {
+	BasicHeader();
+
+	static constexpr auto kTrackEstimatedTime = 0x01U;
+
+	Format getFormat() const {
+		return static_cast<Format>(format);
+	}
+	void setFormat(Format format) {
+		this->format = static_cast<uint32>(format);
+	}
+
+	uint32 format : 8;
+	uint32 flags : 24;
+	uint32 systemTime = 0;
+	uint32 reserved1 = 0;
+	uint32 reserved2 = 0;
+};
+
+struct EstimatedTimePoint {
+	uint32 relative1 = 0;
+	uint32 relative2 = 0;
+	uint32 system = 0;
+
+	void setRelative(uint64 value) {
+		relative1 = uint32(value & 0xFFFFFFFFU);
+		relative2 = uint32((value >> 32) & 0xFFFFFFFFU);
+	}
+	uint64 getRelative() const {
+		return uint64(relative1) | (uint64(relative2) << 32);
+	}
+};
+
+struct Store {
+	static constexpr auto kType = RecordType(0x01);
+
+	void setSize(size_type size);
+	size_type getSize() const;
+
+	RecordType type = kType;
+	uint8 tag = 0;
+	EntrySize size = { { 0 } };
+	PlaceId place = { { 0 } };
+	uint32 checksum = 0;
+	Key key;
+};
+
+struct StoreWithTime : Store {
+	EstimatedTimePoint time;
+	uint32 reserved = 0;
+};
+
+struct MultiStore {
+	static constexpr auto kType = RecordType(0x02);
+
+	explicit MultiStore(size_type count = 0);
+
+	RecordType type = kType;
+	RecordsCount count = { { 0 } };
+	uint32 reserved1 = 0;
+	uint32 reserved2 = 0;
+	uint32 reserved3 = 0;
+
+	using Part = Store;
+	size_type validateCount() const;
+};
+struct MultiStoreWithTime : MultiStore {
+	using MultiStore::MultiStore;
+
+	using Part = StoreWithTime;
+};
+
+struct MultiRemove {
+	static constexpr auto kType = RecordType(0x03);
+
+	explicit MultiRemove(size_type count = 0);
+
+	RecordType type = kType;
+	RecordsCount count = { { 0 } };
+	uint32 reserved1 = 0;
+	uint32 reserved2 = 0;
+	uint32 reserved3 = 0;
+
+	using Part = Key;
+	size_type validateCount() const;
+};
+
+struct MultiAccess {
+	static constexpr auto kType = RecordType(0x04);
+
+	explicit MultiAccess(
+		EstimatedTimePoint time,
+		size_type count = 0);
+
+	RecordType type = kType;
+	RecordsCount count = { { 0 } };
+	EstimatedTimePoint time;
+
+	using Part = Key;
+	size_type validateCount() const;
+};
+
+} // namespace details
+} // namespace Cache
+} // namespace Storage
+
+namespace std {
+
+template <>
+struct hash<Storage::Cache::Key> {
+	size_t operator()(const Storage::Cache::Key &key) const {
+		return (hash<uint64>()(key.high) ^ hash<uint64>()(key.low));
+	}
+};
+
+} // namespace std

Telegram/lib_storage/storage/cache/storage_encrypted_file_tests.cpp

@@ -0,0 +1,246 @@
+// This file is part of Desktop App Toolkit,
+// a set of libraries for developing nice desktop applications.
+//
+// For license and copyright information please follow this link:
+// https://github.com/desktop-app/legal/blob/master/LEGAL
+//
+#include <catch.hpp>
+
+#include "storage/storage_encrypted_file.h"
+
+#include <QtCore/QThread>
+#include <QtCore/QCoreApplication>
+
+#ifdef Q_OS_WIN
+#include "platform/win/windows_dlls.h"
+#endif // Q_OS_WIN
+
+#include <QtCore/QProcess>
+
+#include <thread>
+#ifdef Q_OS_MAC
+#include <mach-o/dyld.h>
+#elif !defined Q_OS_WIN // Q_OS_MAC
+#include <unistd.h>
+#endif // Q_OS_MAC || !Q_OS_WIN
+
+extern int (*TestForkedMethod)();
+
+const auto Key = Storage::EncryptionKey(bytes::make_vector(
+	bytes::make_span("\
+abcdefgh01234567abcdefgh01234567abcdefgh01234567abcdefgh01234567\
+abcdefgh01234567abcdefgh01234567abcdefgh01234567abcdefgh01234567\
+abcdefgh01234567abcdefgh01234567abcdefgh01234567abcdefgh01234567\
+abcdefgh01234567abcdefgh01234567abcdefgh01234567abcdefgh01234567\
+").subspan(0, Storage::EncryptionKey::kSize)));
+
+const auto Name = QString("test.file");
+
+const auto Test1 = bytes::make_span("testbytetestbyte").subspan(0, 16);
+const auto Test2 = bytes::make_span("bytetestbytetest").subspan(0, 16);
+
+struct ForkInit {
+	static int Method() {
+		Storage::File file;
+		const auto result = file.open(
+			Name,
+			Storage::File::Mode::ReadAppend,
+			Key);
+		if (result != Storage::File::Result::Success) {
+			return -1;
+		}
+
+		auto data = bytes::vector(16);
+		const auto read = file.read(data);
+		if (read != data.size()) {
+			return -1;
+		} else if (data != bytes::make_vector(Test1)) {
+			return -1;
+		}
+
+		if (!file.write(data) || !file.flush()) {
+			return -1;
+		}
+#ifdef _DEBUG
+		while (true) {
+			std::this_thread::sleep_for(std::chrono::seconds(1));
+		}
+#else // _DEBUG
+		std::this_thread::sleep_for(std::chrono::seconds(1));
+		return 0;
+#endif // _DEBUG
+	}
+	ForkInit() {
+#ifdef Q_OS_WIN
+		Platform::Dlls::start();
+#endif // Q_OS_WIN
+
+		TestForkedMethod = &ForkInit::Method;
+	}
+
+};
+
+ForkInit ForkInitializer;
+QProcess ForkProcess;
+
+TEST_CASE("simple encrypted file", "[storage_encrypted_file]") {
+	SECTION("writing file") {
+		Storage::File file;
+		const auto result = file.open(
+			Name,
+			Storage::File::Mode::Write,
+			Key);
+		REQUIRE(result == Storage::File::Result::Success);
+
+		auto data = bytes::make_vector(Test1);
+		const auto success = file.write(data);
+		REQUIRE(success);
+	}
+	SECTION("reading and writing file") {
+		Storage::File file;
+		const auto result = file.open(
+			Name,
+			Storage::File::Mode::ReadAppend,
+			Key);
+		REQUIRE(result == Storage::File::Result::Success);
+
+		auto data = bytes::vector(Test1.size());
+		const auto read = file.read(data);
+		REQUIRE(read == data.size());
+		REQUIRE(data == bytes::make_vector(Test1));
+
+		data = bytes::make_vector(Test2);
+		const auto success = file.write(data);
+		REQUIRE(success);
+	}
+	SECTION("offset and seek") {
+		Storage::File file;
+		const auto result = file.open(
+			Name,
+			Storage::File::Mode::ReadAppend,
+			Key);
+		REQUIRE(result == Storage::File::Result::Success);
+		REQUIRE(file.offset() == 0);
+		REQUIRE(file.size() == Test1.size() + Test2.size());
+
+		const auto success1 = file.seek(Test1.size());
+		REQUIRE(success1);
+		REQUIRE(file.offset() == Test1.size());
+
+		auto data = bytes::vector(Test2.size());
+		const auto read = file.read(data);
+		REQUIRE(read == data.size());
+		REQUIRE(data == bytes::make_vector(Test2));
+		REQUIRE(file.offset() == Test1.size() + Test2.size());
+		REQUIRE(file.size() == Test1.size() + Test2.size());
+
+		const auto success2 = file.seek(Test1.size());
+		REQUIRE(success2);
+		REQUIRE(file.offset() == Test1.size());
+
+		data = bytes::make_vector(Test1);
+		const auto success3 = file.write(data) && file.write(data);
+		REQUIRE(success3);
+
+		REQUIRE(file.offset() == 3 * Test1.size());
+		REQUIRE(file.size() == 3 * Test1.size());
+	}
+	SECTION("reading file") {
+		Storage::File file;
+
+		const auto result = file.open(
+			Name,
+			Storage::File::Mode::Read,
+			Key);
+		REQUIRE(result == Storage::File::Result::Success);
+
+		auto data = bytes::vector(32);
+		const auto read = file.read(data);
+		REQUIRE(read == data.size());
+		REQUIRE(data == bytes::concatenate(Test1, Test1));
+	}
+	SECTION("moving file") {
+		const auto result = Storage::File::Move(Name, "other.file");
+		REQUIRE(result);
+	}
+}
+
+TEST_CASE("two process encrypted file", "[storage_encrypted_file]") {
+	SECTION("writing file") {
+		Storage::File file;
+		const auto result = file.open(
+			Name,
+			Storage::File::Mode::Write,
+			Key);
+		REQUIRE(result == Storage::File::Result::Success);
+
+		auto data = bytes::make_vector(Test1);
+		const auto success = file.write(data);
+		REQUIRE(success);
+	}
+	SECTION("access from subprocess") {
+		SECTION("start subprocess") {
+			const auto application = []() -> QString {
+#ifdef Q_OS_WIN
+				return "tests_storage.exe";
+#else // Q_OS_WIN
+				constexpr auto kMaxPath = 1024;
+				char result[kMaxPath] = { 0 };
+				uint32_t size = kMaxPath;
+#ifdef Q_OS_MAC
+				if (_NSGetExecutablePath(result, &size) == 0) {
+					return result;
+				}
+#else // Q_OS_MAC
+				auto count = readlink("/proc/self/exe", result, size);
+				if (count > 0) {
+					return result;
+				}
+#endif // Q_OS_MAC
+				return "tests_storage";
+#endif // Q_OS_WIN
+			}();
+
+			ForkProcess.start(application + " --forked");
+			const auto started = ForkProcess.waitForStarted();
+			REQUIRE(started);
+		}
+		SECTION("read subprocess result") {
+			std::this_thread::sleep_for(std::chrono::milliseconds(500));
+
+			Storage::File file;
+
+			const auto result = file.open(
+				Name,
+				Storage::File::Mode::Read,
+				Key);
+			REQUIRE(result == Storage::File::Result::Success);
+
+			auto data = bytes::vector(32);
+			const auto read = file.read(data);
+			REQUIRE(read == data.size());
+			REQUIRE(data == bytes::concatenate(Test1, Test1));
+		}
+		SECTION("take subprocess result") {
+			REQUIRE(ForkProcess.state() == QProcess::Running);
+
+			Storage::File file;
+
+			const auto result = file.open(
+				Name,
+				Storage::File::Mode::ReadAppend,
+				Key);
+			REQUIRE(result == Storage::File::Result::Success);
+
+			auto data = bytes::vector(32);
+			const auto read = file.read(data);
+			REQUIRE(read == data.size());
+			REQUIRE(data == bytes::concatenate(Test1, Test1));
+
+			const auto finished = ForkProcess.waitForFinished(0);
+			REQUIRE(finished);
+			REQUIRE(ForkProcess.state() == QProcess::NotRunning);
+		}
+	}
+
+}