init

Telegram/SourceFiles/storage/details/storage_file_utilities.cpp

@@ -0,0 +1,677 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#include "storage/details/storage_file_utilities.h"
+
+#include "mtproto/mtproto_auth_key.h"
+#include "base/platform/base_platform_file_utilities.h"
+#include "base/openssl_help.h"
+#include "base/random.h"
+
+#include <crl/crl_object_on_thread.h>
+#include <QtCore/QtEndian>
+#include <QtCore/QSaveFile>
+
+namespace Storage {
+namespace details {
+namespace {
+
+constexpr char TdfMagic[] = { 'T', 'D', 'F', '$' };
+constexpr auto TdfMagicLen = int(sizeof(TdfMagic));
+
+constexpr auto kStrongIterationsCount = 100'000;
+
+struct WriteEntry {
+	QString basePath;
+	QString base;
+	QByteArray data;
+	QByteArray md5;
+};
+
+class WriteManager final {
+public:
+	explicit WriteManager(crl::weak_on_thread<WriteManager> weak);
+
+	void write(WriteEntry &&entry);
+	void writeSync(WriteEntry &&entry);
+	void writeSyncAll();
+
+private:
+	void scheduleWrite();
+	void writeScheduled();
+	bool writeOneScheduledNow();
+	void writeNow(WriteEntry &&entry);
+
+	template <typename File>
+	[[nodiscard]] bool open(File &file, const WriteEntry &entry, char postfix);
+
+	[[nodiscard]] QString path(const WriteEntry &entry, char postfix) const;
+	[[nodiscard]] bool writeHeader(
+		const QString &basePath,
+		QFileDevice &file);
+
+	crl::weak_on_thread<WriteManager> _weak;
+	std::deque<WriteEntry> _scheduled;
+
+};
+
+class AsyncWriteManager final {
+public:
+	void write(WriteEntry &&entry);
+	void writeSync(WriteEntry &&entry);
+	void sync();
+	void stop();
+
+private:
+	std::optional<crl::object_on_thread<WriteManager>> _manager;
+	bool _finished = false;
+
+};
+
+WriteManager::WriteManager(crl::weak_on_thread<WriteManager> weak)
+: _weak(std::move(weak)) {
+}
+
+void WriteManager::write(WriteEntry &&entry) {
+	const auto i = ranges::find(_scheduled, entry.base, &WriteEntry::base);
+	if (i == end(_scheduled)) {
+		_scheduled.push_back(std::move(entry));
+	} else {
+		*i = std::move(entry);
+	}
+	scheduleWrite();
+}
+
+void WriteManager::writeSync(WriteEntry &&entry) {
+	const auto i = ranges::find(_scheduled, entry.base, &WriteEntry::base);
+	if (i != end(_scheduled)) {
+		_scheduled.erase(i);
+	}
+	writeNow(std::move(entry));
+}
+
+void WriteManager::writeNow(WriteEntry &&entry) {
+	const auto path = [&](char postfix) {
+		return this->path(entry, postfix);
+	};
+	const auto open = [&](auto &file, char postfix) {
+		return this->open(file, entry, postfix);
+	};
+	const auto write = [&](auto &file) {
+		file.write(entry.data);
+		file.write(entry.md5);
+	};
+	const auto safe = path('s');
+	const auto simple = path('0');
+	const auto backup = path('1');
+	QSaveFile save;
+	if (open(save, 's')) {
+		write(save);
+		if (save.commit()) {
+			QFile::remove(simple);
+			QFile::remove(backup);
+			return;
+		}
+		LOG(("Storage Error: Could not commit '%1'.").arg(safe));
+	}
+	QFile plain;
+	if (open(plain, '0')) {
+		write(plain);
+		base::Platform::FlushFileData(plain);
+		plain.close();
+
+		QFile::remove(backup);
+		if (base::Platform::RenameWithOverwrite(simple, safe)) {
+			return;
+		}
+		QFile::remove(safe);
+		LOG(("Storage Error: Could not rename '%1' to '%2', removing.").arg(
+			simple,
+			safe));
+	}
+}
+
+void WriteManager::writeSyncAll() {
+	while (writeOneScheduledNow()) {
+	}
+}
+
+bool WriteManager::writeOneScheduledNow() {
+	if (_scheduled.empty()) {
+		return false;
+	}
+
+	auto entry = std::move(_scheduled.front());
+	_scheduled.pop_front();
+
+	writeNow(std::move(entry));
+	return true;
+}
+
+bool WriteManager::writeHeader(const QString &basePath, QFileDevice &file) {
+	if (!file.open(QIODevice::WriteOnly)) {
+		const auto dir = QDir(basePath);
+		if (dir.exists()) {
+			return false;
+		} else if (!QDir().mkpath(dir.absolutePath())) {
+			return false;
+		} else if (!file.open(QIODevice::WriteOnly)) {
+			return false;
+		}
+	}
+	file.write(TdfMagic, TdfMagicLen);
+	const auto version = qint32(AppVersion);
+	file.write((const char*)&version, sizeof(version));
+	return true;
+}
+
+QString WriteManager::path(const WriteEntry &entry, char postfix) const {
+	return entry.base + postfix;
+}
+
+template <typename File>
+bool WriteManager::open(File &file, const WriteEntry &entry, char postfix) {
+	const auto name = path(entry, postfix);
+	file.setFileName(name);
+	if (!writeHeader(entry.basePath, file)) {
+		LOG(("Storage Error: Could not open '%1' for writing.").arg(name));
+		return false;
+	}
+	return true;
+}
+
+void WriteManager::scheduleWrite() {
+	_weak.with([](WriteManager &that) {
+		that.writeScheduled();
+	});
+}
+
+void WriteManager::writeScheduled() {
+	if (writeOneScheduledNow() && !_scheduled.empty()) {
+		scheduleWrite();
+	}
+}
+
+void AsyncWriteManager::write(WriteEntry &&entry) {
+	Expects(!_finished);
+
+	if (!_manager) {
+		_manager.emplace();
+	}
+	_manager->with([entry = std::move(entry)](WriteManager &manager) mutable {
+		manager.write(std::move(entry));
+	});
+}
+
+void AsyncWriteManager::writeSync(WriteEntry &&entry) {
+	Expects(!_finished);
+
+	if (!_manager) {
+		_manager.emplace();
+	}
+	_manager->with_sync([&](WriteManager &manager) {
+		manager.writeSync(std::move(entry));
+	});
+}
+
+void AsyncWriteManager::sync() {
+	if (_manager) {
+		_manager->with_sync([](WriteManager &manager) {
+			manager.writeSyncAll();
+		});
+	}
+}
+
+void AsyncWriteManager::stop() {
+	if (_manager) {
+		sync();
+		_manager.reset();
+	}
+	_finished = true;
+}
+
+AsyncWriteManager Manager;
+
+} // namespace
+
+QString ToFilePart(FileKey val) {
+	QString result;
+	result.reserve(0x10);
+	for (int32 i = 0; i < 0x10; ++i) {
+		uchar v = (val & 0x0F);
+		result.push_back((v < 0x0A) ? QChar('0' + v) : QChar('A' + (v - 0x0A)));
+		val >>= 4;
+	}
+	return result;
+}
+
+bool KeyAlreadyUsed(QString &name) {
+	name += '0';
+	if (QFileInfo::exists(name)) {
+		return true;
+	}
+	name[name.size() - 1] = '1';
+	if (QFileInfo::exists(name)) {
+		return true;
+	}
+	name[name.size() - 1] = 's';
+	if (QFileInfo::exists(name)) {
+		return true;
+	}
+	return false;
+}
+
+FileKey GenerateKey(const QString &basePath) {
+	FileKey result;
+	QString path;
+	path.reserve(basePath.size() + 0x11);
+	path += basePath;
+	do {
+		result = base::RandomValue<FileKey>();
+		path.resize(basePath.size());
+		path += ToFilePart(result);
+	} while (!result || KeyAlreadyUsed(path));
+
+	return result;
+}
+
+void ClearKey(const FileKey &key, const QString &basePath) {
+	QString name;
+	name.reserve(basePath.size() + 0x11);
+	name.append(basePath).append(ToFilePart(key)).append('0');
+	QFile::remove(name);
+	name[name.size() - 1] = '1';
+	QFile::remove(name);
+	name[name.size() - 1] = 's';
+	QFile::remove(name);
+}
+
+bool CheckStreamStatus(QDataStream &stream) {
+	if (stream.status() != QDataStream::Ok) {
+		LOG(("Bad data stream status: %1").arg(stream.status()));
+		return false;
+	}
+	return true;
+}
+
+MTP::AuthKeyPtr CreateLocalKey(
+		const QByteArray &passcode,
+		const QByteArray &salt) {
+	const auto s = bytes::make_span(salt);
+	const auto hash = openssl::Sha512(s, bytes::make_span(passcode), s);
+	const auto iterationsCount = passcode.isEmpty()
+		? 1 // Don't slow down for no password.
+		: kStrongIterationsCount;
+
+	auto key = MTP::AuthKey::Data{ { gsl::byte{} } };
+	PKCS5_PBKDF2_HMAC(
+		reinterpret_cast<const char*>(hash.data()),
+		hash.size(),
+		reinterpret_cast<const unsigned char*>(s.data()),
+		s.size(),
+		iterationsCount,
+		EVP_sha512(),
+		key.size(),
+		reinterpret_cast<unsigned char*>(key.data()));
+	return std::make_shared<MTP::AuthKey>(key);
+}
+
+MTP::AuthKeyPtr CreateLegacyLocalKey(
+		const QByteArray &passcode,
+		const QByteArray &salt) {
+	auto key = MTP::AuthKey::Data{ { gsl::byte{} } };
+	const auto iterationsCount = passcode.isEmpty()
+		? LocalEncryptNoPwdIterCount // Don't slow down for no password.
+		: LocalEncryptIterCount;
+
+	PKCS5_PBKDF2_HMAC_SHA1(
+		passcode.constData(),
+		passcode.size(),
+		(uchar*)salt.data(),
+		salt.size(),
+		iterationsCount,
+		key.size(),
+		(uchar*)key.data());
+
+	return std::make_shared<MTP::AuthKey>(key);
+}
+
+FileReadDescriptor::~FileReadDescriptor() {
+	if (version) {
+		stream.setDevice(nullptr);
+		if (buffer.isOpen()) {
+			buffer.close();
+		}
+		buffer.setBuffer(nullptr);
+	}
+}
+
+EncryptedDescriptor::EncryptedDescriptor() {
+}
+
+EncryptedDescriptor::EncryptedDescriptor(uint32 size) {
+	uint32 fullSize = sizeof(uint32) + size;
+	if (fullSize & 0x0F) fullSize += 0x10 - (fullSize & 0x0F);
+	data.reserve(fullSize);
+
+	data.resize(sizeof(uint32));
+	buffer.setBuffer(&data);
+	buffer.open(QIODevice::WriteOnly);
+	buffer.seek(sizeof(uint32));
+	stream.setDevice(&buffer);
+	stream.setVersion(QDataStream::Qt_5_1);
+}
+
+EncryptedDescriptor::~EncryptedDescriptor() {
+	finish();
+}
+
+void EncryptedDescriptor::finish() {
+	if (stream.device()) stream.setDevice(nullptr);
+	if (buffer.isOpen()) buffer.close();
+	buffer.setBuffer(nullptr);
+}
+
+FileWriteDescriptor::FileWriteDescriptor(
+	const FileKey &key,
+	const QString &basePath,
+	bool sync)
+: FileWriteDescriptor(ToFilePart(key), basePath, sync) {
+}
+
+FileWriteDescriptor::FileWriteDescriptor(
+	const QString &name,
+	const QString &basePath,
+	bool sync)
+: _basePath(basePath)
+, _sync(sync) {
+	init(name);
+}
+
+FileWriteDescriptor::~FileWriteDescriptor() {
+	finish();
+}
+
+void FileWriteDescriptor::init(const QString &name) {
+	_base = _basePath + name;
+	_buffer.setBuffer(&_safeData);
+	const auto opened = _buffer.open(QIODevice::WriteOnly);
+	Assert(opened);
+	_stream.setDevice(&_buffer);
+}
+
+void FileWriteDescriptor::writeData(const QByteArray &data) {
+	if (!_stream.device()) {
+		return;
+	}
+	_stream << data;
+	quint32 len = data.isNull() ? 0xffffffff : data.size();
+	if (QSysInfo::ByteOrder != QSysInfo::BigEndian) {
+		len = qbswap(len);
+	}
+	_md5.feed(&len, sizeof(len));
+	_md5.feed(data.constData(), data.size());
+	_fullSize += sizeof(len) + data.size();
+}
+
+void FileWriteDescriptor::writeEncrypted(
+	EncryptedDescriptor &data,
+	const MTP::AuthKeyPtr &key) {
+	writeData(PrepareEncrypted(data, key));
+}
+
+void FileWriteDescriptor::finish() {
+	if (!_stream.device()) {
+		return;
+	}
+
+	_stream.setDevice(nullptr);
+	_md5.feed(&_fullSize, sizeof(_fullSize));
+	qint32 version = AppVersion;
+	_md5.feed(&version, sizeof(version));
+	_md5.feed(TdfMagic, TdfMagicLen);
+
+	_buffer.close();
+
+	auto entry = WriteEntry{
+		.basePath = _basePath,
+		.base = _base,
+		.data = _safeData,
+		.md5 = QByteArray((const char*)_md5.result(), 0x10)
+	};
+	if (_sync) {
+		Manager.writeSync(std::move(entry));
+	} else {
+		Manager.write(std::move(entry));
+	}
+}
+
+[[nodiscard]] QByteArray PrepareEncrypted(
+		EncryptedDescriptor &data,
+		const MTP::AuthKeyPtr &key) {
+	data.finish();
+	QByteArray &toEncrypt(data.data);
+
+	// prepare for encryption
+	uint32 size = toEncrypt.size(), fullSize = size;
+	if (fullSize & 0x0F) {
+		fullSize += 0x10 - (fullSize & 0x0F);
+		toEncrypt.resize(fullSize);
+		base::RandomFill(toEncrypt.data() + size, fullSize - size);
+	}
+	*(uint32*)toEncrypt.data() = size;
+	QByteArray encrypted(0x10 + fullSize, Qt::Uninitialized); // 128bit of sha1 - key128, sizeof(data), data
+	hashSha1(toEncrypt.constData(), toEncrypt.size(), encrypted.data());
+	MTP::aesEncryptLocal(toEncrypt.constData(), encrypted.data() + 0x10, fullSize, key, encrypted.constData());
+
+	return encrypted;
+}
+
+bool ReadFile(
+		FileReadDescriptor &result,
+		const QString &name,
+		const QString &basePath) {
+	const auto base = basePath + name;
+
+	// detect order of read attempts
+	QString toTry[2];
+	const auto modern = base + 's';
+	if (QFileInfo::exists(modern)) {
+		toTry[0] = modern;
+	} else {
+		// Legacy way.
+		toTry[0] = base + '0';
+		QFileInfo toTry0(toTry[0]);
+		if (toTry0.exists()) {
+			toTry[1] = basePath + name + '1';
+			QFileInfo toTry1(toTry[1]);
+			if (toTry1.exists()) {
+				QDateTime mod0 = toTry0.lastModified();
+				QDateTime mod1 = toTry1.lastModified();
+				if (mod0 < mod1) {
+					qSwap(toTry[0], toTry[1]);
+				}
+			} else {
+				toTry[1] = QString();
+			}
+		} else {
+			toTry[0][toTry[0].size() - 1] = '1';
+		}
+	}
+	for (int32 i = 0; i < 2; ++i) {
+		QString fname(toTry[i]);
+		if (fname.isEmpty()) break;
+
+		QFile f(fname);
+		if (!f.open(QIODevice::ReadOnly)) {
+			DEBUG_LOG(("App Info: failed to open '%1' for reading"
+				).arg(name));
+			continue;
+		}
+
+		// check magic
+		char magic[TdfMagicLen];
+		if (f.read(magic, TdfMagicLen) != TdfMagicLen) {
+			DEBUG_LOG(("App Info: failed to read magic from '%1'"
+				).arg(name));
+			continue;
+		}
+		if (memcmp(magic, TdfMagic, TdfMagicLen)) {
+			DEBUG_LOG(("App Info: bad magic %1 in '%2'").arg(
+				Logs::mb(magic, TdfMagicLen).str(),
+				name));
+			continue;
+		}
+
+		// read app version
+		qint32 version;
+		if (f.read((char*)&version, sizeof(version)) != sizeof(version)) {
+			DEBUG_LOG(("App Info: failed to read version from '%1'"
+				).arg(name));
+			continue;
+		}
+		if (version > AppVersion) {
+			DEBUG_LOG(("App Info: version too big %1 for '%2', my version %3"
+				).arg(version
+				).arg(name
+				).arg(AppVersion));
+			continue;
+		}
+
+		// read data
+		QByteArray bytes = f.read(f.size());
+		int32 dataSize = bytes.size() - 16;
+		if (dataSize < 0) {
+			DEBUG_LOG(("App Info: bad file '%1', could not read sign part"
+				).arg(name));
+			continue;
+		}
+
+		// check signature
+		HashMd5 md5;
+		md5.feed(bytes.constData(), dataSize);
+		md5.feed(&dataSize, sizeof(dataSize));
+		md5.feed(&version, sizeof(version));
+		md5.feed(magic, TdfMagicLen);
+		if (memcmp(md5.result(), bytes.constData() + dataSize, 16)) {
+			DEBUG_LOG(("App Info: bad file '%1', signature did not match"
+				).arg(name));
+			continue;
+		}
+
+		bytes.resize(dataSize);
+		result.data = bytes;
+		bytes = QByteArray();
+
+		result.version = version;
+		result.buffer.setBuffer(&result.data);
+		result.buffer.open(QIODevice::ReadOnly);
+		result.stream.setDevice(&result.buffer);
+		result.stream.setVersion(QDataStream::Qt_5_1);
+
+		if ((i == 0 && !toTry[1].isEmpty()) || i == 1) {
+			QFile::remove(toTry[1 - i]);
+		}
+
+		return true;
+	}
+	return false;
+}
+
+bool DecryptLocal(
+		EncryptedDescriptor &result,
+		const QByteArray &encrypted,
+		const MTP::AuthKeyPtr &key) {
+	if (encrypted.size() <= 16 || (encrypted.size() & 0x0F)) {
+		LOG(("App Error: bad encrypted part size: %1").arg(encrypted.size()));
+		return false;
+	}
+	uint32 fullLen = encrypted.size() - 16;
+
+	QByteArray decrypted;
+	decrypted.resize(fullLen);
+	const char *encryptedKey = encrypted.constData(), *encryptedData = encrypted.constData() + 16;
+	aesDecryptLocal(encryptedData, decrypted.data(), fullLen, key, encryptedKey);
+	uchar sha1Buffer[20];
+	if (memcmp(hashSha1(decrypted.constData(), decrypted.size(), sha1Buffer), encryptedKey, 16)) {
+		LOG(("App Info: bad decrypt key, data not decrypted - incorrect password?"));
+		return false;
+	}
+
+	uint32 dataLen = *(const uint32*)decrypted.constData();
+	if (dataLen > uint32(decrypted.size()) || dataLen <= fullLen - 16 || dataLen < sizeof(uint32)) {
+		LOG(("App Error: bad decrypted part size: %1, fullLen: %2, decrypted size: %3").arg(dataLen).arg(fullLen).arg(decrypted.size()));
+		return false;
+	}
+
+	decrypted.resize(dataLen);
+	result.data = decrypted;
+	decrypted = QByteArray();
+
+	result.buffer.setBuffer(&result.data);
+	result.buffer.open(QIODevice::ReadOnly);
+	result.buffer.seek(sizeof(uint32)); // skip len
+	result.stream.setDevice(&result.buffer);
+	result.stream.setVersion(QDataStream::Qt_5_1);
+
+	return true;
+}
+
+bool ReadEncryptedFile(
+		FileReadDescriptor &result,
+		const QString &name,
+		const QString &basePath,
+		const MTP::AuthKeyPtr &key) {
+	if (!ReadFile(result, name, basePath)) {
+		return false;
+	}
+	QByteArray encrypted;
+	result.stream >> encrypted;
+
+	EncryptedDescriptor data;
+	if (!DecryptLocal(data, encrypted, key)) {
+		result.stream.setDevice(nullptr);
+		if (result.buffer.isOpen()) result.buffer.close();
+		result.buffer.setBuffer(nullptr);
+		result.data = QByteArray();
+		result.version = 0;
+		return false;
+	}
+
+	result.stream.setDevice(0);
+	if (result.buffer.isOpen()) {
+		result.buffer.close();
+	}
+	result.buffer.setBuffer(0);
+	result.data = data.data;
+	result.buffer.setBuffer(&result.data);
+	result.buffer.open(QIODevice::ReadOnly);
+	result.buffer.seek(data.buffer.pos());
+	result.stream.setDevice(&result.buffer);
+	result.stream.setVersion(QDataStream::Qt_5_1);
+
+	return true;
+}
+
+bool ReadEncryptedFile(
+		FileReadDescriptor &result,
+		const FileKey &fkey,
+		const QString &basePath,
+		const MTP::AuthKeyPtr &key) {
+	return ReadEncryptedFile(result, ToFilePart(fkey), basePath, key);
+}
+
+void Sync() {
+	Manager.sync();
+}
+
+void Finish() {
+	Manager.stop();
+}
+
+} // namespace details
+} // namespace Storage

Telegram/SourceFiles/storage/details/storage_file_utilities.h

@@ -0,0 +1,113 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#pragma once
+
+#include "storage/storage_account.h"
+
+#include <QtCore/QBuffer>
+
+namespace Storage {
+namespace details {
+
+[[nodiscard]] QString ToFilePart(FileKey val);
+[[nodiscard]] bool KeyAlreadyUsed(QString &name);
+[[nodiscard]] FileKey GenerateKey(const QString &basePath);
+void ClearKey(const FileKey &key, const QString &basePath);
+
+[[nodiscard]] bool CheckStreamStatus(QDataStream &stream);
+[[nodiscard]] MTP::AuthKeyPtr CreateLocalKey(
+	const QByteArray &passcode,
+	const QByteArray &salt);
+[[nodiscard]] MTP::AuthKeyPtr CreateLegacyLocalKey(
+	const QByteArray &passcode,
+	const QByteArray &salt);
+
+struct FileReadDescriptor final {
+	~FileReadDescriptor();
+
+	int32 version = 0;
+	QByteArray data;
+	QBuffer buffer;
+	QDataStream stream;
+};
+
+struct EncryptedDescriptor final {
+	EncryptedDescriptor();
+	explicit EncryptedDescriptor(uint32 size);
+	~EncryptedDescriptor();
+
+	void finish();
+
+	QByteArray data;
+	QBuffer buffer;
+	QDataStream stream;
+};
+
+[[nodiscard]] QByteArray PrepareEncrypted(
+	EncryptedDescriptor &data,
+	const MTP::AuthKeyPtr &key);
+
+class FileWriteDescriptor final {
+public:
+	FileWriteDescriptor(
+		const FileKey &key,
+		const QString &basePath,
+		bool sync = false);
+	FileWriteDescriptor(
+		const QString &name,
+		const QString &basePath,
+		bool sync = false);
+	~FileWriteDescriptor();
+
+	void writeData(const QByteArray &data);
+	void writeEncrypted(
+		EncryptedDescriptor &data,
+		const MTP::AuthKeyPtr &key);
+
+private:
+	void init(const QString &name);
+	void finish();
+
+	const QString _basePath;
+	QBuffer _buffer;
+	QDataStream _stream;
+	QByteArray _safeData;
+	QString _base;
+	HashMd5 _md5;
+	int _fullSize = 0;
+	bool _sync = false;
+
+};
+
+bool ReadFile(
+	FileReadDescriptor &result,
+	const QString &name,
+	const QString &basePath);
+
+bool DecryptLocal(
+	EncryptedDescriptor &result,
+	const QByteArray &encrypted,
+	const MTP::AuthKeyPtr &key);
+
+bool ReadEncryptedFile(
+	FileReadDescriptor &result,
+	const QString &name,
+	const QString &basePath,
+	const MTP::AuthKeyPtr &key);
+
+bool ReadEncryptedFile(
+	FileReadDescriptor &result,
+	const FileKey &fkey,
+	const QString &basePath,
+	const MTP::AuthKeyPtr &key);
+
+void Sync();
+void Finish();
+
+} // namespace details
+} // namespace Storage

Telegram/SourceFiles/storage/details/storage_settings_scheme.h

@@ -0,0 +1,178 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#pragma once
+
+#include "mtproto/mtproto_dc_options.h"
+#include "main/main_session_settings.h"
+#include "storage/storage_account.h"
+
+namespace MTP {
+class AuthKey;
+} // namespace MTP
+
+namespace Storage {
+namespace details {
+
+struct ReadSettingsContext {
+	[[nodiscard]] Main::SessionSettings &sessionSettings() {
+		if (!sessionSettingsStorage) {
+			sessionSettingsStorage
+				= std::make_unique<Main::SessionSettings>();
+		}
+		return *sessionSettingsStorage;
+	}
+
+	// This field is read in ReadSetting.
+	bool legacyHasCustomDayBackground = false;
+
+	// Those fields are written in ReadSetting.
+	MTP::DcOptions fallbackConfigLegacyDcOptions
+		= MTP::DcOptions(MTP::Environment::Production);
+	qint32 fallbackConfigLegacyChatSizeMax = 0;
+	qint32 fallbackConfigLegacySavedGifsLimit = 0;
+	qint32 fallbackConfigLegacyStickersRecentLimit = 0;
+	qint32 fallbackConfigLegacyStickersFavedLimit = 0;
+	qint32 fallbackConfigLegacyMegagroupSizeMax = 0;
+	QString fallbackConfigLegacyTxtDomainString;
+	QByteArray fallbackConfig;
+
+	qint64 cacheTotalSizeLimit = 0;
+	qint32 cacheTotalTimeLimit = 0;
+	qint64 cacheBigFileTotalSizeLimit = 0;
+	qint32 cacheBigFileTotalTimeLimit = 0;
+
+	std::unique_ptr<Main::SessionSettings> sessionSettingsStorage;
+
+	FileKey themeKeyLegacy = 0;
+	FileKey themeKeyDay = 0;
+	FileKey themeKeyNight = 0;
+	FileKey backgroundKeyDay = 0;
+	FileKey backgroundKeyNight = 0;
+	bool backgroundKeysRead = false;
+	bool tileDay = false;
+	bool tileNight = true;
+	bool tileRead = false;
+	FileKey langPackKey = 0;
+	FileKey languagesKey = 0;
+
+	QByteArray mtpAuthorization;
+	std::vector<std::shared_ptr<MTP::AuthKey>> mtpLegacyKeys;
+	qint32 mtpLegacyMainDcId = 0;
+	qint32 mtpLegacyUserId = 0;
+
+	bool legacyRead = false;
+};
+
+[[nodiscard]] bool ReadSetting(
+	quint32 blockId,
+	QDataStream &stream,
+	int version,
+	ReadSettingsContext &context);
+
+void ApplyReadFallbackConfig(ReadSettingsContext &context);
+
+enum {
+	dbiKey = 0x00,
+	dbiUser = 0x01,
+	dbiDcOptionOldOld = 0x02,
+	dbiChatSizeMaxOld = 0x03,
+	dbiMutePeerOld = 0x04,
+	dbiSendKeyOld = 0x05,
+	dbiAutoStart = 0x06,
+	dbiStartMinimized = 0x07,
+	dbiSoundFlashBounceNotifyOld = 0x08,
+	dbiWorkModeOld = 0x09,
+	dbiSeenTrayTooltip = 0x0a,
+	dbiDesktopNotifyOld = 0x0b,
+	dbiAutoUpdate = 0x0c,
+	dbiLastUpdateCheck = 0x0d,
+	dbiWindowPositionOld = 0x0e,
+	dbiConnectionTypeOldOld = 0x0f,
+	// 0x10 reserved
+	dbiDefaultAttach = 0x11,
+	dbiCatsAndDogsOld = 0x12,
+	dbiReplaceEmojiOld = 0x13,
+	dbiAskDownloadPathOld = 0x14,
+	dbiDownloadPathOldOld = 0x15,
+	dbiScaleOld = 0x16,
+	dbiEmojiTabOld = 0x17,
+	dbiRecentEmojiOldOldOld = 0x18,
+	dbiLoggedPhoneNumberOld = 0x19,
+	dbiMutedPeersOld = 0x1a,
+	// 0x1b reserved
+	dbiNotifyViewOld = 0x1c,
+	dbiSendToMenu = 0x1d,
+	dbiCompressPastedImageOld = 0x1e,
+	dbiLangOld = 0x1f,
+	dbiLangFileOld = 0x20,
+	dbiTileBackgroundOld = 0x21,
+	dbiAutoLockOld = 0x22,
+	dbiDialogLastPath = 0x23,
+	dbiRecentEmojiOldOld = 0x24,
+	dbiEmojiVariantsOldOld = 0x25,
+	dbiRecentStickers = 0x26,
+	dbiDcOptionOld = 0x27,
+	dbiTryIPv6Old = 0x28,
+	dbiSongVolumeOld = 0x29,
+	dbiWindowsNotificationsOld = 0x30,
+	dbiIncludeMutedOld = 0x31,
+	dbiMegagroupSizeMaxOld = 0x32,
+	dbiDownloadPathOld = 0x33,
+	dbiAutoDownloadOld = 0x34,
+	dbiSavedGifsLimitOld = 0x35,
+	dbiShowingSavedGifsOld = 0x36,
+	dbiAutoPlayOld = 0x37,
+	dbiAdaptiveForWideOld = 0x38,
+	dbiHiddenPinnedMessagesOld = 0x39,
+	dbiRecentEmojiOld = 0x3a,
+	dbiEmojiVariantsOld = 0x3b,
+	dbiDialogsModeOld = 0x40,
+	dbiModerateModeOld = 0x41,
+	dbiVideoVolumeOld = 0x42,
+	dbiStickersRecentLimitOld = 0x43,
+	dbiNativeNotificationsOld = 0x44,
+	dbiNotificationsCountOld = 0x45,
+	dbiNotificationsCornerOld = 0x46,
+	dbiThemeKeyOld = 0x47,
+	dbiDialogsWidthRatioOld = 0x48,
+	dbiUseExternalVideoPlayerOld = 0x49,
+	dbiDcOptionsOld = 0x4a,
+	dbiMtpAuthorization = 0x4b,
+	dbiLastSeenWarningSeenOld = 0x4c,
+	dbiSessionSettings = 0x4d,
+	dbiLangPackKey = 0x4e,
+	dbiConnectionTypeOld = 0x4f,
+	dbiStickersFavedLimitOld = 0x50,
+	dbiSuggestStickersByEmojiOld = 0x51,
+	dbiSuggestEmojiOld = 0x52,
+	dbiTxtDomainStringOldOld = 0x53,
+	dbiThemeKey = 0x54,
+	dbiTileBackground = 0x55,
+	dbiCacheSettingsOld = 0x56,
+	dbiPowerSaving = 0x57,
+	dbiScalePercent = 0x58,
+	dbiPlaybackSpeedOld = 0x59,
+	dbiLanguagesKey = 0x5a,
+	dbiCallSettingsOld = 0x5b,
+	dbiCacheSettings = 0x5c,
+	dbiTxtDomainStringOld = 0x5d,
+	dbiApplicationSettings = 0x5e,
+	dbiDialogsFiltersOld = 0x5f,
+	dbiFallbackProductionConfig = 0x60,
+	dbiBackgroundKey = 0x61,
+
+	dbiEncryptedWithSalt = 333,
+	dbiEncrypted = 444,
+
+	// 500-600 reserved
+
+	dbiVersion = 666,
+};
+
+} // namespace details
+} // namespace Storage

Telegram/SourceFiles/storage/download_manager_mtproto.h

@@ -0,0 +1,288 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#pragma once
+
+#include "data/data_file_origin.h"
+#include "base/timer.h"
+#include "base/weak_ptr.h"
+
+class ApiWrap;
+
+namespace MTP {
+class Error;
+} // namespace MTP
+
+namespace Storage {
+
+// Different part sizes are not supported for now :(
+// Because we start downloading with some part size
+// and then we get a CDN-redirect where we support only
+// fixed part size download for hash checking.
+constexpr auto kDownloadPartSize = 128 * 1024;
+
+class DownloadMtprotoTask;
+
+class DownloadManagerMtproto final : public base::has_weak_ptr {
+public:
+	using Task = DownloadMtprotoTask;
+
+	explicit DownloadManagerMtproto(not_null<ApiWrap*> api);
+	~DownloadManagerMtproto();
+
+	[[nodiscard]] ApiWrap &api() const {
+		return *_api;
+	}
+
+	void enqueue(not_null<Task*> task, int priority);
+	void remove(not_null<Task*> task);
+
+	void notifyTaskFinished() {
+		_taskFinished.fire({});
+	}
+	[[nodiscard]] rpl::producer<> taskFinished() const {
+		return _taskFinished.events();
+	}
+
+	int changeRequestedAmount(MTP::DcId dcId, int index, int delta);
+	void requestSucceeded(
+		MTP::DcId dcId,
+		int index,
+		int amountAtRequestStart,
+		crl::time timeAtRequestStart);
+	void checkSendNextAfterSuccess(MTP::DcId dcId);
+	[[nodiscard]] int chooseSessionIndex(MTP::DcId dcId) const;
+
+	void notifyNonPremiumDelay(DocumentId id) {
+		_nonPremiumDelays.fire_copy(id);
+	}
+	[[nodiscard]] rpl::producer<DocumentId> nonPremiumDelays() const {
+		return _nonPremiumDelays.events();
+	}
+
+private:
+	class Queue final {
+	public:
+		void enqueue(not_null<Task*> task, int priority);
+		void remove(not_null<Task*> task);
+		void resetGeneration();
+		[[nodiscard]] bool empty() const;
+		[[nodiscard]] Task *nextTask(bool onlyHighestPriority) const;
+		void removeSession(int index);
+
+	private:
+		struct Enqueued {
+			not_null<Task*> task;
+			int priority = 0;
+		};
+		std::vector<Enqueued> _tasks;
+
+	};
+	struct DcSessionBalanceData {
+		DcSessionBalanceData();
+
+		int requested = 0;
+		int successes = 0; // Since last timeout in this dc in any session.
+		int maxWaitedAmount = 0;
+	};
+	struct DcBalanceData {
+		DcBalanceData();
+
+		std::vector<DcSessionBalanceData> sessions;
+		crl::time lastSessionRemove = 0;
+		int sessionRemoveIndex = 0;
+		int sessionRemoveTimes = 0;
+		int timeouts = 0; // Since all sessions had successes >= required.
+		int totalRequested = 0;
+	};
+
+	void checkSendNext();
+	void checkSendNext(MTP::DcId dcId, Queue &queue);
+	bool trySendNextPart(MTP::DcId dcId, Queue &queue);
+
+	void killSessionsSchedule(MTP::DcId dcId);
+	void killSessionsCancel(MTP::DcId dcId);
+	void killSessions();
+	void killSessions(MTP::DcId dcId);
+
+	void resetGeneration();
+	void sessionTimedOut(MTP::DcId dcId, int index);
+	void removeSession(MTP::DcId dcId);
+
+	const not_null<ApiWrap*> _api;
+
+	rpl::event_stream<> _taskFinished;
+	rpl::event_stream<DocumentId> _nonPremiumDelays;
+
+	base::flat_map<MTP::DcId, DcBalanceData> _balanceData;
+	base::Timer _resetGenerationTimer;
+
+	base::flat_map<MTP::DcId, crl::time> _killSessionsWhen;
+	base::Timer _killSessionsTimer;
+
+	base::flat_map<MTP::DcId, Queue> _queues;
+	rpl::lifetime _lifetime;
+
+};
+
+class DownloadMtprotoTask : public base::has_weak_ptr {
+public:
+	struct Location {
+		std::variant<
+			StorageFileLocation,
+			WebFileLocation,
+			GeoPointLocation,
+			AudioAlbumThumbLocation> data;
+	};
+
+	DownloadMtprotoTask(
+		not_null<DownloadManagerMtproto*> owner,
+		const StorageFileLocation &location,
+		Data::FileOrigin origin);
+	DownloadMtprotoTask(
+		not_null<DownloadManagerMtproto*> owner,
+		MTP::DcId dcId,
+		const Location &location);
+	virtual ~DownloadMtprotoTask();
+
+	[[nodiscard]] MTP::DcId dcId() const;
+	[[nodiscard]] Data::FileOrigin fileOrigin() const;
+	[[nodiscard]] uint64 objectId() const;
+	[[nodiscard]] const Location &location() const;
+
+	[[nodiscard]] virtual bool readyToRequest() const = 0;
+	void loadPart(int sessionIndex);
+	void removeSession(int sessionIndex);
+
+	void refreshFileReferenceFrom(
+		const Data::UpdatedFileReferences &updates,
+		int requestId,
+		const QByteArray &current);
+
+protected:
+	[[nodiscard]] bool haveSentRequests() const;
+	[[nodiscard]] bool haveSentRequestForOffset(int64 offset) const;
+	void cancelAllRequests();
+	void cancelRequestForOffset(int64 offset);
+
+	void addToQueue(int priority = 0);
+	void removeFromQueue();
+
+	[[nodiscard]] ApiWrap &api() const {
+		return _owner->api();
+	}
+
+private:
+	struct RequestData {
+		int64 offset = 0;
+		mutable int sessionIndex = 0;
+		int requestedInSession = 0;
+		crl::time sent = 0;
+
+		inline bool operator<(const RequestData &other) const {
+			return offset < other.offset;
+		}
+	};
+	struct CdnFileHash {
+		CdnFileHash(int limit, QByteArray hash) : limit(limit), hash(hash) {
+		}
+		int limit = 0;
+		QByteArray hash;
+	};
+	enum class CheckCdnHashResult {
+		NoHash,
+		Invalid,
+		Good,
+	};
+	enum class FinishRequestReason {
+		Success,
+		Redirect,
+		Cancel,
+	};
+
+	// Called only if readyToRequest() == true.
+	[[nodiscard]] virtual int64 takeNextRequestOffset() = 0;
+	virtual bool feedPart(int64 offset, const QByteArray &bytes) = 0;
+	virtual bool setWebFileSizeHook(int64 size);
+	virtual void cancelOnFail() = 0;
+
+	void cancelRequest(mtpRequestId requestId);
+	void makeRequest(const RequestData &requestData);
+	void normalPartLoaded(
+		const MTPupload_File &result,
+		mtpRequestId requestId);
+	void webPartLoaded(
+		const MTPupload_WebFile &result,
+		mtpRequestId requestId);
+	void cdnPartLoaded(
+		const MTPupload_CdnFile &result,
+		mtpRequestId requestId);
+	void reuploadDone(
+		const MTPVector<MTPFileHash> &result,
+		mtpRequestId requestId);
+	void requestMoreCdnFileHashes();
+	void getCdnFileHashesDone(
+		const MTPVector<MTPFileHash> &result,
+		mtpRequestId requestId);
+
+	void partLoaded(int64 offset, const QByteArray &bytes);
+
+	bool partFailed(const MTP::Error &error, mtpRequestId requestId);
+	bool normalPartFailed(
+		QByteArray fileReference,
+		const MTP::Error &error,
+		mtpRequestId requestId);
+	bool cdnPartFailed(const MTP::Error &error, mtpRequestId requestId);
+
+	[[nodiscard]] mtpRequestId sendRequest(const RequestData &requestData);
+	void placeSentRequest(
+		mtpRequestId requestId,
+		const RequestData &requestData);
+	[[nodiscard]] RequestData finishSentRequest(
+		mtpRequestId requestId,
+		FinishRequestReason reason);
+	void switchToCDN(
+		const RequestData &requestData,
+		const MTPDupload_fileCdnRedirect &redirect);
+	void addCdnHashes(const QVector<MTPFileHash> &hashes);
+	void changeCDNParams(
+		const RequestData &requestData,
+		MTP::DcId dcId,
+		const QByteArray &token,
+		const QByteArray &encryptionKey,
+		const QByteArray &encryptionIV,
+		const QVector<MTPFileHash> &hashes);
+
+	[[nodiscard]] CheckCdnHashResult checkCdnFileHash(
+		int64 offset,
+		bytes::const_span buffer);
+
+	void subscribeToNonPremiumLimit();
+
+	const not_null<DownloadManagerMtproto*> _owner;
+	const MTP::DcId _dcId = 0;
+
+	// _location can be changed with an updated file_reference.
+	Location _location;
+	const Data::FileOrigin _origin;
+
+	base::flat_map<mtpRequestId, RequestData> _sentRequests;
+	base::flat_map<int64, mtpRequestId> _requestByOffset;
+
+	MTP::DcId _cdnDcId = 0;
+	QByteArray _cdnToken;
+	QByteArray _cdnEncryptionKey;
+	QByteArray _cdnEncryptionIV;
+	base::flat_map<int64, CdnFileHash> _cdnFileHashes;
+	base::flat_map<RequestData, QByteArray> _cdnUncheckedParts;
+	mtpRequestId _cdnHashesRequestId = 0;
+
+	rpl::lifetime _nonPremiumLimitSubscription;
+
+};
+
+} // namespace Storage

Telegram/SourceFiles/storage/file_download.cpp

@@ -0,0 +1,553 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#include "storage/file_download.h"
+
+#include "data/data_document.h"
+#include "data/data_session.h"
+#include "data/data_file_origin.h"
+#include "mainwidget.h"
+#include "mainwindow.h"
+#include "core/application.h"
+#include "core/file_location.h"
+#include "storage/storage_account.h"
+#include "storage/file_download_mtproto.h"
+#include "storage/file_download_web.h"
+#include "platform/platform_file_utilities.h"
+#include "main/main_session.h"
+#include "apiwrap.h"
+#include "core/crash_reports.h"
+#include "base/bytes.h"
+
+namespace {
+
+class FromMemoryLoader final : public FileLoader {
+public:
+	FromMemoryLoader(
+		not_null<Main::Session*> session,
+		const QByteArray &data,
+		const QString &toFile,
+		int64 loadSize,
+		int64 fullSize,
+		LocationType locationType,
+		LoadToCacheSetting toCache,
+		LoadFromCloudSetting fromCloud,
+		bool autoLoading,
+		uint8 cacheTag);
+
+private:
+	Storage::Cache::Key cacheKey() const override;
+	std::optional<MediaKey> fileLocationKey() const override;
+	void cancelHook() override;
+	void startLoading() override;
+
+	QByteArray _data;
+
+};
+
+FromMemoryLoader::FromMemoryLoader(
+	not_null<Main::Session*> session,
+	const QByteArray &data,
+	const QString &toFile,
+	int64 loadSize,
+	int64 fullSize,
+	LocationType locationType,
+	LoadToCacheSetting toCache,
+	LoadFromCloudSetting fromCloud,
+	bool autoLoading,
+	uint8 cacheTag
+) : FileLoader(
+	session,
+	toFile,
+	loadSize,
+	fullSize,
+	locationType,
+	toCache,
+	fromCloud,
+	autoLoading,
+	cacheTag)
+, _data(data) {
+}
+
+Storage::Cache::Key FromMemoryLoader::cacheKey() const {
+	return {};
+}
+
+std::optional<MediaKey> FromMemoryLoader::fileLocationKey() const {
+	return std::nullopt;
+}
+
+void FromMemoryLoader::cancelHook() {
+}
+
+void FromMemoryLoader::startLoading() {
+	finishWithBytes(_data);
+}
+
+} // namespace
+
+FileLoader::FileLoader(
+	not_null<Main::Session*> session,
+	const QString &toFile,
+	int64 loadSize,
+	int64 fullSize,
+	LocationType locationType,
+	LoadToCacheSetting toCache,
+	LoadFromCloudSetting fromCloud,
+	bool autoLoading,
+	uint8 cacheTag)
+: _session(session)
+, _autoLoading(autoLoading)
+, _cacheTag(cacheTag)
+, _filename(toFile)
+, _file(_filename)
+, _toCache(toCache)
+, _fromCloud(fromCloud)
+, _loadSize(loadSize)
+, _fullSize(fullSize)
+, _locationType(locationType) {
+	Expects(_loadSize <= _fullSize);
+	Expects(!_filename.isEmpty() || (_fullSize <= Storage::kMaxFileInMemory));
+}
+
+FileLoader::~FileLoader() {
+	Expects(_finished);
+}
+
+Main::Session &FileLoader::session() const {
+	return *_session;
+}
+
+void FileLoader::finishWithBytes(const QByteArray &data) {
+	_data = data;
+	_localStatus = LocalStatus::Loaded;
+	if (!_filename.isEmpty() && _toCache == LoadToCacheAsWell) {
+		if (!_fileIsOpen) _fileIsOpen = _file.open(QIODevice::WriteOnly);
+		if (!_fileIsOpen) {
+			cancel(FailureReason::FileWriteFailure);
+			return;
+		}
+		_file.seek(0);
+		if (_file.write(_data) != qint64(_data.size())) {
+			cancel(FailureReason::FileWriteFailure);
+			return;
+		}
+	}
+
+	_finished = true;
+	if (_fileIsOpen) {
+		_file.close();
+		_fileIsOpen = false;
+		Platform::File::PostprocessDownloaded(
+			QFileInfo(_file).absoluteFilePath());
+	}
+	const auto session = _session;
+	_updates.fire_done();
+	session->notifyDownloaderTaskFinished();
+}
+
+QImage FileLoader::imageData(int progressiveSizeLimit) const {
+	if (_imageData.isNull() && _locationType == UnknownFileLocation) {
+		readImage(progressiveSizeLimit);
+	}
+	return _imageData;
+}
+
+void FileLoader::readImage(int progressiveSizeLimit) const {
+	const auto buffer = progressiveSizeLimit
+		? QByteArray::fromRawData(_data.data(), progressiveSizeLimit)
+		: _data;
+	auto read = Images::Read({ .content = buffer });
+	if (!read.image.isNull()) {
+		_imageData = std::move(read.image);
+		_imageFormat = read.format;
+	}
+}
+
+Data::FileOrigin FileLoader::fileOrigin() const {
+	return Data::FileOrigin();
+}
+
+float64 FileLoader::currentProgress() const {
+	return _finished
+		? 1.
+		: !_loadSize
+		? 0.
+		: std::clamp(float64(currentOffset()) / _loadSize, 0., 1.);
+}
+
+bool FileLoader::setFileName(const QString &fileName) {
+	if (_toCache != LoadToCacheAsWell || !_filename.isEmpty()) {
+		return fileName.isEmpty() || (fileName == _filename);
+	}
+	_filename = fileName;
+	_file.setFileName(_filename);
+	return true;
+}
+
+void FileLoader::permitLoadFromCloud() {
+	_fromCloud = LoadFromCloudOrLocal;
+}
+
+void FileLoader::increaseLoadSize(int64 size, bool autoLoading) {
+	Expects(size > _loadSize);
+	Expects(size <= _fullSize);
+
+	_loadSize = size;
+	_autoLoading = autoLoading;
+}
+
+void FileLoader::notifyAboutProgress() {
+	_updates.fire({});
+}
+
+void FileLoader::localLoaded(
+		const StorageImageSaved &result,
+		const QByteArray &imageFormat,
+		const QImage &imageData) {
+	_localLoading = nullptr;
+	if (result.data.isEmpty()) {
+		_localStatus = LocalStatus::NotFound;
+		start();
+		return;
+	}
+	const auto partial = result.data.startsWith("partial:");
+	constexpr auto kPrefix = 8;
+	if (partial	&& result.data.size() < _loadSize + kPrefix) {
+		_localStatus = LocalStatus::NotFound;
+		if (checkForOpen()) {
+			startLoadingWithPartial(result.data);
+		}
+		return;
+	}
+	if (!imageData.isNull()) {
+		_imageFormat = imageFormat;
+		_imageData = imageData;
+	}
+	finishWithBytes(partial
+		? QByteArray::fromRawData(
+			result.data.data() + kPrefix,
+			result.data.size() - kPrefix)
+		: result.data);
+}
+
+void FileLoader::start() {
+	if (_finished || tryLoadLocal()) {
+		return;
+	} else if (_fromCloud == LoadFromLocalOnly) {
+		cancel();
+		return;
+	}
+
+	if (checkForOpen()) {
+		startLoading();
+	}
+}
+
+bool FileLoader::checkForOpen() {
+	if (_filename.isEmpty()
+		|| (_toCache != LoadToFileOnly)
+		|| _fileIsOpen) {
+		return true;
+	}
+	_fileIsOpen = _file.open(QIODevice::WriteOnly);
+	if (_fileIsOpen) {
+		return true;
+	}
+	cancel(FailureReason::FileWriteFailure);
+	return false;
+}
+
+void FileLoader::loadLocal(const Storage::Cache::Key &key) {
+	const auto readImage = (_locationType != AudioFileLocation);
+	auto done = [=, guard = _localLoading.make_guard()](
+			QByteArray &&value,
+			QImage &&image,
+			QByteArray &&format) mutable {
+		crl::on_main(std::move(guard), [
+			=,
+			value = std::move(value),
+			image = std::move(image),
+			format = std::move(format)
+		]() mutable {
+			localLoaded(
+				StorageImageSaved(std::move(value)),
+				format,
+				std::move(image));
+		});
+	};
+	_session->data().cache().get(key, [=, callback = std::move(done)](
+			QByteArray &&value) mutable {
+		if (readImage && !value.startsWith("partial:")) {
+			crl::async([
+				value = std::move(value),
+				done = std::move(callback)
+			]() mutable {
+				auto read = Images::Read({ .content = value });
+				if (!read.image.isNull()) {
+					done(
+						std::move(value),
+						std::move(read.image),
+						std::move(read.format));
+				} else {
+					done(std::move(value), {}, {});
+				}
+			});
+		} else {
+			callback(std::move(value), {}, {});
+		}
+	});
+}
+
+bool FileLoader::tryLoadLocal() {
+	if (_localStatus == LocalStatus::NotFound
+		|| _localStatus == LocalStatus::Loaded) {
+		return false;
+	} else if (_localStatus == LocalStatus::Loading) {
+		return true;
+	}
+
+	if (_toCache == LoadToCacheAsWell) {
+		const auto key = cacheKey();
+		if (key.low || key.high) {
+			loadLocal(key);
+			notifyAboutProgress();
+		}
+	}
+	if (_localStatus != LocalStatus::NotTried) {
+		return _finished;
+	} else if (_localLoading) {
+		_localStatus = LocalStatus::Loading;
+		return true;
+	}
+	_localStatus = LocalStatus::NotFound;
+	return false;
+}
+
+void FileLoader::cancel() {
+	cancel(FailureReason::NoFailure);
+}
+
+void FileLoader::cancel(FailureReason fail) {
+	const auto started = (currentOffset() > 0);
+
+	cancelHook();
+
+	_cancelled = true;
+	_finished = true;
+	if (_fileIsOpen) {
+		_file.close();
+		_fileIsOpen = false;
+		_file.remove();
+	}
+	_data = QByteArray();
+
+	const auto weak = base::make_weak(this);
+	if (fail != FailureReason::NoFailure) {
+		_updates.fire_error_copy({ fail, started });
+	} else {
+		_updates.fire_done();
+	}
+	if (weak) {
+		_filename = QString();
+		_file.setFileName(_filename);
+	}
+}
+
+int64 FileLoader::currentOffset() const {
+	return (_fileIsOpen ? _file.size() : _data.size()) - _skippedBytes;
+}
+
+bool FileLoader::writeResultPart(int64 offset, bytes::const_span buffer) {
+	Expects(!_finished);
+
+	if (buffer.empty()) {
+		return true;
+	}
+	if (_fileIsOpen) {
+		auto fsize = _file.size();
+		if (offset < fsize) {
+			_skippedBytes -= buffer.size();
+		} else if (offset > fsize) {
+			_skippedBytes += offset - fsize;
+		}
+		_file.seek(offset);
+		if (_file.write(reinterpret_cast<const char*>(buffer.data()), buffer.size()) != qint64(buffer.size())) {
+			cancel(FailureReason::FileWriteFailure);
+			return false;
+		}
+		return true;
+	}
+	_data.reserve(offset + buffer.size());
+	if (offset > _data.size()) {
+		_skippedBytes += offset - _data.size();
+		_data.resize(offset);
+	}
+	if (offset == _data.size()) {
+		_data.append(reinterpret_cast<const char*>(buffer.data()), buffer.size());
+	} else {
+		_skippedBytes -= buffer.size();
+		if (int64(offset + buffer.size()) > _data.size()) {
+			_data.resize(offset + buffer.size());
+		}
+		const auto dst = bytes::make_detached_span(_data).subspan(
+			offset,
+			buffer.size());
+		bytes::copy(dst, buffer);
+	}
+	return true;
+}
+
+QByteArray FileLoader::readLoadedPartBack(int64 offset, int size) {
+	Expects(offset >= 0 && size > 0);
+
+	if (_fileIsOpen) {
+		if (_file.openMode() == QIODevice::WriteOnly) {
+			_file.close();
+			_fileIsOpen = _file.open(QIODevice::ReadWrite);
+			if (!_fileIsOpen) {
+				cancel(FailureReason::FileWriteFailure);
+				return QByteArray();
+			}
+		}
+		if (!_file.seek(offset)) {
+			return QByteArray();
+		}
+		auto result = _file.read(size);
+		return (result.size() == size) ? result : QByteArray();
+	}
+	return (offset + size <= _data.size())
+		? _data.mid(offset, size)
+		: QByteArray();
+}
+
+bool FileLoader::finalizeResult() {
+	Expects(!_finished);
+
+	if (!_filename.isEmpty() && (_toCache == LoadToCacheAsWell)) {
+		if (!_fileIsOpen) {
+			_fileIsOpen = _file.open(QIODevice::WriteOnly);
+		}
+		_file.seek(0);
+		if (!_fileIsOpen || _file.write(_data) != qint64(_data.size())) {
+			cancel(FailureReason::FileWriteFailure);
+			return false;
+		}
+	}
+
+	_finished = true;
+	if (_fileIsOpen) {
+		_file.close();
+		_fileIsOpen = false;
+		Platform::File::PostprocessDownloaded(
+			QFileInfo(_file).absoluteFilePath());
+	}
+	if (_localStatus == LocalStatus::NotFound) {
+		if (const auto key = fileLocationKey()) {
+			if (!_filename.isEmpty()) {
+				_session->local().writeFileLocation(
+					*key,
+					Core::FileLocation(_filename));
+			}
+		}
+		const auto key = cacheKey();
+		if ((_toCache == LoadToCacheAsWell)
+			&& (_data.size() <= Storage::kMaxFileInMemory)
+			&& (key.low || key.high)) {
+			_session->data().cache().put(
+				cacheKey(),
+				Storage::Cache::Database::TaggedValue(
+					base::duplicate((!_fullSize || _data.size() == _fullSize)
+						? _data
+						: ("partial:" + _data)),
+					_cacheTag));
+		}
+	}
+	const auto session = _session;
+	_updates.fire_done();
+	session->notifyDownloaderTaskFinished();
+	return true;
+}
+
+std::unique_ptr<FileLoader> CreateFileLoader(
+		not_null<Main::Session*> session,
+		const DownloadLocation &location,
+		Data::FileOrigin origin,
+		const QString &toFile,
+		int64 loadSize,
+		int64 fullSize,
+		LocationType locationType,
+		LoadToCacheSetting toCache,
+		LoadFromCloudSetting fromCloud,
+		bool autoLoading,
+		uint8 cacheTag) {
+	auto result = std::unique_ptr<FileLoader>();
+	v::match(location.data, [&](const StorageFileLocation &data) {
+		result = std::make_unique<mtpFileLoader>(
+			session,
+			data,
+			origin,
+			locationType,
+			toFile,
+			loadSize,
+			fullSize,
+			toCache,
+			fromCloud,
+			autoLoading,
+			cacheTag);
+	}, [&](const WebFileLocation &data) {
+		result = std::make_unique<mtpFileLoader>(
+			session,
+			data,
+			loadSize,
+			fullSize,
+			fromCloud,
+			autoLoading,
+			cacheTag);
+	}, [&](const GeoPointLocation &data) {
+		result = std::make_unique<mtpFileLoader>(
+			session,
+			data,
+			loadSize,
+			fullSize,
+			fromCloud,
+			autoLoading,
+			cacheTag);
+	}, [&](const PlainUrlLocation &data) {
+		result = std::make_unique<webFileLoader>(
+			session,
+			data.url,
+			toFile,
+			fromCloud,
+			autoLoading,
+			cacheTag);
+	}, [&](const AudioAlbumThumbLocation &data) {
+		result = std::make_unique<mtpFileLoader>(
+			session,
+			data,
+			loadSize,
+			fullSize,
+			fromCloud,
+			autoLoading,
+			cacheTag);
+	}, [&](const InMemoryLocation &data) {
+		result = std::make_unique<FromMemoryLoader>(
+			session,
+			data.bytes,
+			toFile,
+			loadSize,
+			fullSize,
+			locationType,
+			toCache,
+			LoadFromCloudOrLocal,
+			autoLoading,
+			cacheTag);
+	});
+
+	Ensures(result != nullptr);
+	return result;
+}

Telegram/SourceFiles/storage/file_download.h

@@ -0,0 +1,206 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#pragma once
+
+#include "base/binary_guard.h"
+#include "base/weak_ptr.h"
+
+#include <QtNetwork/QNetworkReply>
+
+namespace Data {
+struct FileOrigin;
+} // namespace Data
+
+namespace Main {
+class Session;
+} // namespace Main
+
+namespace Storage {
+namespace Cache {
+struct Key;
+} // namespace Cache
+
+// 10 MB max file could be hold in memory
+// This value is used in local cache database settings!
+constexpr auto kMaxFileInMemory = 10 * 1024 * 1024;
+
+// 2 MB stickers hold in memory, auto loaded and displayed inline
+constexpr auto kMaxStickerBytesSize = 2 * 1024 * 1024;
+
+// 10 MB GIF and mp4 animations held in memory while playing
+constexpr auto kMaxWallPaperInMemory = kMaxFileInMemory;
+constexpr auto kMaxAnimationInMemory = kMaxFileInMemory;
+
+// 4096x4096 is max area.
+constexpr auto kMaxWallPaperDimension = 4096;
+
+} // namespace Storage
+
+struct StorageImageSaved {
+	StorageImageSaved() = default;
+	explicit StorageImageSaved(const QByteArray &data) : data(data) {
+	}
+
+	QByteArray data;
+
+};
+
+class FileLoader : public base::has_weak_ptr {
+public:
+	enum class FailureReason {
+		NoFailure,
+		FileWriteFailure,
+		OtherFailure,
+	};
+
+	struct Error {
+		FailureReason failureReason = FailureReason::NoFailure;
+		bool started = false;
+	};
+
+	FileLoader(
+		not_null<Main::Session*> session,
+		const QString &toFile,
+		int64 loadSize,
+		int64 fullSize,
+		LocationType locationType,
+		LoadToCacheSetting toCache,
+		LoadFromCloudSetting fromCloud,
+		bool autoLoading,
+		uint8 cacheTag);
+	virtual ~FileLoader();
+
+	[[nodiscard]] Main::Session &session() const;
+
+	[[nodiscard]] bool finished() const {
+		return _finished;
+	}
+	void finishWithBytes(const QByteArray &data);
+	[[nodiscard]] bool cancelled() const {
+		return _cancelled;
+	}
+	[[nodiscard]] const QByteArray &bytes() const {
+		return _data;
+	}
+	[[nodiscard]] virtual uint64 objId() const {
+		return 0;
+	}
+	[[nodiscard]] QImage imageData(int progressiveSizeLimit = 0) const;
+	[[nodiscard]] QString fileName() const {
+		return _filename;
+	}
+	// Used in MainWidget::documentLoadFailed.
+	[[nodiscard]] virtual Data::FileOrigin fileOrigin() const;
+	[[nodiscard]] float64 currentProgress() const;
+	[[nodiscard]] virtual int64 currentOffset() const;
+	[[nodiscard]] int64 fullSize() const {
+		return _fullSize;
+	}
+	[[nodiscard]] int64 loadSize() const {
+		return _loadSize;
+	}
+
+	bool setFileName(const QString &filename); // set filename for loaders to cache
+	void permitLoadFromCloud();
+	void increaseLoadSize(int64 size, bool autoLoading);
+
+	void start();
+	void cancel();
+
+	[[nodiscard]] bool loadingLocal() const {
+		return (_localStatus == LocalStatus::Loading);
+	}
+	[[nodiscard]] bool autoLoading() const {
+		return _autoLoading;
+	}
+
+	void localLoaded(
+		const StorageImageSaved &result,
+		const QByteArray &imageFormat,
+		const QImage &imageData);
+
+	[[nodiscard]] rpl::producer<rpl::empty_value, Error> updates() const {
+		return _updates.events();
+	}
+
+	[[nodiscard]] rpl::lifetime &lifetime() {
+		return _lifetime;
+	}
+
+protected:
+	enum class LocalStatus {
+		NotTried,
+		NotFound,
+		Loading,
+		Loaded,
+	};
+
+	void readImage(int progressiveSizeLimit) const;
+
+	bool checkForOpen();
+	bool tryLoadLocal();
+	void loadLocal(const Storage::Cache::Key &key);
+	virtual Storage::Cache::Key cacheKey() const = 0;
+	virtual std::optional<MediaKey> fileLocationKey() const = 0;
+	virtual void cancelHook() = 0;
+	virtual void startLoading() = 0;
+	virtual void startLoadingWithPartial(const QByteArray &data) {
+		startLoading();
+	}
+
+	void cancel(FailureReason failed);
+
+	void notifyAboutProgress();
+
+	bool writeResultPart(int64 offset, bytes::const_span buffer);
+	bool finalizeResult();
+	[[nodiscard]] QByteArray readLoadedPartBack(int64 offset, int size);
+
+	const not_null<Main::Session*> _session;
+
+	bool _autoLoading = false;
+	uint8 _cacheTag = 0;
+	bool _finished = false;
+	bool _cancelled = false;
+	mutable LocalStatus _localStatus = LocalStatus::NotTried;
+
+	QString _filename;
+	QFile _file;
+	bool _fileIsOpen = false;
+
+	LoadToCacheSetting _toCache;
+	LoadFromCloudSetting _fromCloud;
+
+	QByteArray _data;
+
+	int64 _loadSize = 0;
+	int64 _fullSize = 0;
+	int64 _skippedBytes = 0;
+	LocationType _locationType = LocationType();
+
+	base::binary_guard _localLoading;
+	mutable QByteArray _imageFormat;
+	mutable QImage _imageData;
+
+	rpl::lifetime _lifetime;
+	rpl::event_stream<rpl::empty_value, Error> _updates;
+
+};
+
+[[nodiscard]] std::unique_ptr<FileLoader> CreateFileLoader(
+	not_null<Main::Session*> session,
+	const DownloadLocation &location,
+	Data::FileOrigin origin,
+	const QString &toFile,
+	int64 loadSize,
+	int64 fullSize,
+	LocationType locationType,
+	LoadToCacheSetting toCache,
+	LoadFromCloudSetting fromCloud,
+	bool autoLoading,
+	uint8 cacheTag);

Telegram/SourceFiles/storage/file_download_mtproto.cpp

@@ -0,0 +1,221 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#include "storage/file_download_mtproto.h"
+
+#include "data/data_document.h"
+#include "data/data_file_origin.h"
+#include "storage/cache/storage_cache_types.h"
+#include "main/main_session.h"
+#include "apiwrap.h"
+#include "mtproto/mtp_instance.h"
+#include "mtproto/mtproto_config.h"
+#include "mtproto/mtproto_auth_key.h"
+
+mtpFileLoader::mtpFileLoader(
+	not_null<Main::Session*> session,
+	const StorageFileLocation &location,
+	Data::FileOrigin origin,
+	LocationType type,
+	const QString &to,
+	int64 loadSize,
+	int64 fullSize,
+	LoadToCacheSetting toCache,
+	LoadFromCloudSetting fromCloud,
+	bool autoLoading,
+	uint8 cacheTag)
+: FileLoader(
+	session,
+	to,
+	loadSize,
+	fullSize,
+	type,
+	toCache,
+	fromCloud,
+	autoLoading,
+	cacheTag)
+, DownloadMtprotoTask(&session->downloader(), location, origin) {
+}
+
+mtpFileLoader::mtpFileLoader(
+	not_null<Main::Session*> session,
+	const WebFileLocation &location,
+	int64 loadSize,
+	int64 fullSize,
+	LoadFromCloudSetting fromCloud,
+	bool autoLoading,
+	uint8 cacheTag)
+: FileLoader(
+	session,
+	QString(),
+	loadSize,
+	fullSize,
+	UnknownFileLocation,
+	LoadToCacheAsWell,
+	fromCloud,
+	autoLoading,
+	cacheTag)
+, DownloadMtprotoTask(
+	&session->downloader(),
+	session->serverConfig().webFileDcId,
+	{ location }) {
+}
+
+mtpFileLoader::mtpFileLoader(
+	not_null<Main::Session*> session,
+	const GeoPointLocation &location,
+	int64 loadSize,
+	int64 fullSize,
+	LoadFromCloudSetting fromCloud,
+	bool autoLoading,
+	uint8 cacheTag)
+: FileLoader(
+	session,
+	QString(),
+	loadSize,
+	fullSize,
+	UnknownFileLocation,
+	LoadToCacheAsWell,
+	fromCloud,
+	autoLoading,
+	cacheTag)
+, DownloadMtprotoTask(
+	&session->downloader(),
+	session->serverConfig().webFileDcId,
+	{ location }) {
+}
+
+mtpFileLoader::mtpFileLoader(
+	not_null<Main::Session*> session,
+	const AudioAlbumThumbLocation &location,
+	int64 loadSize,
+	int64 fullSize,
+	LoadFromCloudSetting fromCloud,
+	bool autoLoading,
+	uint8 cacheTag)
+: FileLoader(
+	session,
+	QString(),
+	loadSize,
+	fullSize,
+	UnknownFileLocation,
+	LoadToCacheAsWell,
+	fromCloud,
+	autoLoading,
+	cacheTag)
+, DownloadMtprotoTask(
+	&session->downloader(),
+	session->serverConfig().webFileDcId,
+	{ location }) {
+}
+
+mtpFileLoader::~mtpFileLoader() {
+	if (!_finished) {
+		cancel();
+	}
+}
+
+Data::FileOrigin mtpFileLoader::fileOrigin() const {
+	return DownloadMtprotoTask::fileOrigin();
+}
+
+uint64 mtpFileLoader::objId() const {
+	return DownloadMtprotoTask::objectId();
+}
+
+bool mtpFileLoader::readyToRequest() const {
+	return !_finished
+		&& !_lastComplete
+		&& (_fullSize != 0 || !haveSentRequests())
+		&& (!_fullSize || _nextRequestOffset < _loadSize);
+}
+
+int64 mtpFileLoader::takeNextRequestOffset() {
+	Expects(readyToRequest());
+
+	const auto result = _nextRequestOffset;
+	_nextRequestOffset += Storage::kDownloadPartSize;
+	return result;
+}
+
+bool mtpFileLoader::feedPart(int64 offset, const QByteArray &bytes) {
+	const auto buffer = bytes::make_span(bytes);
+	if (!writeResultPart(offset, buffer)) {
+		return false;
+	}
+	if (buffer.empty() || (buffer.size() % 1024)) { // bad next offset
+		_lastComplete = true;
+	}
+	const auto finished = !haveSentRequests()
+		&& (_lastComplete || (_fullSize && _nextRequestOffset >= _loadSize));
+	if (finished) {
+		removeFromQueue();
+		if (!finalizeResult()) {
+			return false;
+		}
+	} else {
+		notifyAboutProgress();
+	}
+	return true;
+}
+
+void mtpFileLoader::cancelOnFail() {
+	cancel(FailureReason::OtherFailure);
+}
+
+bool mtpFileLoader::setWebFileSizeHook(int64 size) {
+	if (!_fullSize || _fullSize == size) {
+		_fullSize = _loadSize = size;
+		return true;
+	}
+	LOG(("MTP Error: "
+		"Bad size provided by bot for webDocument: %1, real: %2"
+		).arg(_fullSize
+		).arg(size));
+	cancel(FailureReason::OtherFailure);
+	return false;
+}
+
+void mtpFileLoader::startLoading() {
+	addToQueue();
+}
+
+void mtpFileLoader::startLoadingWithPartial(const QByteArray &data) {
+	Expects(data.startsWith("partial:"));
+
+	constexpr auto kPrefix = 8;
+	const auto parts = (data.size() - kPrefix) / Storage::kDownloadPartSize;
+	const auto use = parts * int64(Storage::kDownloadPartSize);
+	if (use > 0) {
+		_nextRequestOffset = use;
+		feedPart(0, QByteArray::fromRawData(data.data() + kPrefix, use));
+	}
+	startLoading();
+}
+
+void mtpFileLoader::cancelHook() {
+	cancelAllRequests();
+}
+
+Storage::Cache::Key mtpFileLoader::cacheKey() const {
+	return v::match(location().data, [&](const WebFileLocation &location) {
+		return Data::WebDocumentCacheKey(location);
+	}, [&](const GeoPointLocation &location) {
+		return Data::GeoPointCacheKey(location);
+	}, [&](const StorageFileLocation &location) {
+		return location.cacheKey();
+	}, [&](const AudioAlbumThumbLocation &location) {
+		return Data::AudioAlbumThumbCacheKey(location);
+	});
+}
+
+std::optional<MediaKey> mtpFileLoader::fileLocationKey() const {
+	if (_locationType != UnknownFileLocation) {
+		return mediaKey(_locationType, dcId(), objId());
+	}
+	return std::nullopt;
+}

Telegram/SourceFiles/storage/file_download_mtproto.h

@@ -0,0 +1,74 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#pragma once
+
+#include "storage/file_download.h"
+#include "storage/download_manager_mtproto.h"
+
+class mtpFileLoader final
+	: public FileLoader
+	, private Storage::DownloadMtprotoTask {
+public:
+	mtpFileLoader(
+		not_null<Main::Session*> session,
+		const StorageFileLocation &location,
+		Data::FileOrigin origin,
+		LocationType type,
+		const QString &toFile,
+		int64 loadSize,
+		int64 fullSize,
+		LoadToCacheSetting toCache,
+		LoadFromCloudSetting fromCloud,
+		bool autoLoading,
+		uint8 cacheTag);
+	mtpFileLoader(
+		not_null<Main::Session*> session,
+		const WebFileLocation &location,
+		int64 loadSize,
+		int64 fullSize,
+		LoadFromCloudSetting fromCloud,
+		bool autoLoading,
+		uint8 cacheTag);
+	mtpFileLoader(
+		not_null<Main::Session*> session,
+		const GeoPointLocation &location,
+		int64 loadSize,
+		int64 fullSize,
+		LoadFromCloudSetting fromCloud,
+		bool autoLoading,
+		uint8 cacheTag);
+	mtpFileLoader(
+		not_null<Main::Session*> session,
+		const AudioAlbumThumbLocation &location,
+		int64 loadSize,
+		int64 fullSize,
+		LoadFromCloudSetting fromCloud,
+		bool autoLoading,
+		uint8 cacheTag);
+	~mtpFileLoader();
+
+	Data::FileOrigin fileOrigin() const override;
+	uint64 objId() const override;
+
+private:
+	Storage::Cache::Key cacheKey() const override;
+	std::optional<MediaKey> fileLocationKey() const override;
+	void startLoading() override;
+	void startLoadingWithPartial(const QByteArray &data) override;
+	void cancelHook() override;
+
+	bool readyToRequest() const override;
+	int64 takeNextRequestOffset() override;
+	bool feedPart(int64 offset, const QByteArray &bytes) override;
+	void cancelOnFail() override;
+	bool setWebFileSizeHook(int64 size) override;
+
+	bool _lastComplete = false;
+	int64 _nextRequestOffset = 0;
+
+};

Telegram/SourceFiles/storage/file_download_web.cpp

@@ -0,0 +1,620 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#include "storage/file_download_web.h"
+
+#include "storage/cache/storage_cache_types.h"
+#include "base/timer.h"
+#include "base/weak_ptr.h"
+
+#include <QtNetwork/QAuthenticator>
+
+namespace {
+
+constexpr auto kMaxWebFileQueries = 8;
+constexpr auto kMaxHttpRedirects = 5;
+constexpr auto kResetDownloadPrioritiesTimeout = crl::time(200);
+constexpr auto kMaxWebFile = 4000 * int64(1024 * 1024);
+
+std::weak_ptr<WebLoadManager> GlobalLoadManager;
+
+[[nodiscard]] std::shared_ptr<WebLoadManager> GetManager() {
+	auto result = GlobalLoadManager.lock();
+	if (!result) {
+		GlobalLoadManager = result = std::make_shared<WebLoadManager>();
+	}
+	return result;
+}
+
+enum class ProcessResult {
+	Error,
+	Progress,
+	Finished,
+};
+
+enum class Error {
+};
+
+struct Progress {
+	qint64 ready = 0;
+	qint64 total = 0;
+	QByteArray streamed;
+};
+
+using Update = std::variant<Progress, QByteArray, Error>;
+
+struct UpdateForLoader {
+	not_null<webFileLoader*> loader;
+	Update data;
+};
+
+} // namespace
+
+class WebLoadManager final : public base::has_weak_ptr {
+public:
+	WebLoadManager();
+	~WebLoadManager();
+
+	void enqueue(not_null<webFileLoader*> loader);
+	void remove(not_null<webFileLoader*> loader);
+
+	[[nodiscard]] rpl::producer<Update> updates(
+		not_null<webFileLoader*> loader) const;
+
+private:
+	struct Enqueued {
+		int id = 0;
+		QString url;
+		bool stream = false;
+	};
+	struct Sent {
+		QString url;
+		not_null<QNetworkReply*> reply;
+		bool stream = false;
+		QByteArray data;
+		int64 ready = 0;
+		int64 total = 0;
+		int redirectsLeft = kMaxHttpRedirects;
+	};
+
+	// Constructor.
+	void handleNetworkErrors();
+
+	// Worker thread.
+	void enqueue(int id, const QString &url, bool stream);
+	void remove(int id);
+	void resetGeneration();
+	void checkSendNext();
+	void send(const Enqueued &entry);
+	[[nodiscard]] not_null<QNetworkReply*> send(int id, const QString &url);
+	[[nodiscard]] Sent *findSent(int id, not_null<QNetworkReply*> reply);
+	void removeSent(int id);
+	void progress(
+		int id,
+		not_null<QNetworkReply*> reply,
+		int64 ready,
+		int64 total);
+	void failed(
+		int id,
+		not_null<QNetworkReply*> reply,
+		QNetworkReply::NetworkError error);
+	void redirect(int id, not_null<QNetworkReply*> reply);
+	void notify(
+		int id,
+		not_null<QNetworkReply*> reply,
+		int64 ready,
+		int64 total);
+	void failed(int id, not_null<QNetworkReply*> reply);
+	void finished(int id, not_null<QNetworkReply*> reply);
+	void deleteDeferred(not_null<QNetworkReply*> reply);
+	void queueProgressUpdate(
+		int id,
+		int64 ready,
+		int64 total,
+		QByteArray streamed);
+	void queueFailedUpdate(int id);
+	void queueFinishedUpdate(int id, const QByteArray &data);
+	void clear();
+
+	// Main thread.
+	void sendUpdate(int id, Update &&data);
+
+	QThread _thread;
+	std::unique_ptr<QNetworkAccessManager> _network;
+	base::Timer _resetGenerationTimer;
+
+	// Main thread.
+	rpl::event_stream<UpdateForLoader> _updates;
+	int _autoincrement = 0;
+	base::flat_map<not_null<webFileLoader*>, int> _ids;
+
+	// Worker thread.
+	std::deque<Enqueued> _queue;
+	std::deque<Enqueued> _previousGeneration;
+	base::flat_map<int, Sent> _sent;
+	std::vector<QPointer<QNetworkReply>> _repliesBeingDeleted;
+
+};
+
+WebLoadManager::WebLoadManager()
+: _network(std::make_unique<QNetworkAccessManager>())
+, _resetGenerationTimer(&_thread, [=] { resetGeneration(); }) {
+	handleNetworkErrors();
+
+	_network->moveToThread(&_thread);
+	QObject::connect(&_thread, &QThread::finished, [=] {
+		clear();
+		_network = nullptr;
+	});
+	_thread.start();
+}
+
+void WebLoadManager::handleNetworkErrors() {
+	const auto fail = [=](QNetworkReply *reply) {
+		for (const auto &[id, sent] : _sent) {
+			if (sent.reply == reply) {
+				failed(id, reply);
+				return;
+			}
+		}
+	};
+	QObject::connect(
+		_network.get(),
+		&QNetworkAccessManager::authenticationRequired,
+		fail);
+	QObject::connect(
+		_network.get(),
+		&QNetworkAccessManager::sslErrors,
+		fail);
+}
+
+WebLoadManager::~WebLoadManager() {
+	_thread.quit();
+	_thread.wait();
+}
+
+[[nodiscard]] rpl::producer<Update> WebLoadManager::updates(
+		not_null<webFileLoader*> loader) const {
+	return _updates.events(
+	) | rpl::filter([=](const UpdateForLoader &update) {
+		return (update.loader == loader);
+	}) | rpl::map([=](UpdateForLoader &&update) {
+		return std::move(update.data);
+	});
+}
+
+void WebLoadManager::enqueue(not_null<webFileLoader*> loader) {
+	const auto id = [&] {
+		const auto i = _ids.find(loader);
+		return (i != end(_ids))
+			? i->second
+			: _ids.emplace(loader, ++_autoincrement).first->second;
+	}();
+	const auto url = loader->url();
+	const auto stream = loader->streamLoading();
+	InvokeQueued(_network.get(), [=] {
+		enqueue(id, url, stream);
+	});
+}
+
+void WebLoadManager::remove(not_null<webFileLoader*> loader) {
+	const auto i = _ids.find(loader);
+	if (i == end(_ids)) {
+		return;
+	}
+	const auto id = i->second;
+	_ids.erase(i);
+	InvokeQueued(_network.get(), [=] {
+		remove(id);
+	});
+}
+
+void WebLoadManager::enqueue(int id, const QString &url, bool stream) {
+	const auto i = ranges::find(_queue, id, &Enqueued::id);
+	if (i != end(_queue)) {
+		return;
+	}
+	_previousGeneration.erase(
+		ranges::remove(_previousGeneration, id, &Enqueued::id),
+		end(_previousGeneration));
+	_queue.push_back(Enqueued{ id, url, stream });
+	if (!_resetGenerationTimer.isActive()) {
+		_resetGenerationTimer.callOnce(kResetDownloadPrioritiesTimeout);
+	}
+	checkSendNext();
+}
+
+void WebLoadManager::remove(int id) {
+	_queue.erase(ranges::remove(_queue, id, &Enqueued::id), end(_queue));
+	_previousGeneration.erase(
+		ranges::remove(_previousGeneration, id, &Enqueued::id),
+		end(_previousGeneration));
+	removeSent(id);
+}
+
+void WebLoadManager::resetGeneration() {
+	if (!_previousGeneration.empty()) {
+		std::copy(
+			begin(_previousGeneration),
+			end(_previousGeneration),
+			std::back_inserter(_queue));
+		_previousGeneration.clear();
+	}
+	std::swap(_queue, _previousGeneration);
+}
+
+void WebLoadManager::checkSendNext() {
+	if (_sent.size() >= kMaxWebFileQueries
+		|| (_queue.empty() && _previousGeneration.empty())) {
+		return;
+	}
+	const auto entry = _queue.empty()
+		? _previousGeneration.front()
+		: _queue.front();
+	(_queue.empty() ? _previousGeneration : _queue).pop_front();
+	send(entry);
+}
+
+void WebLoadManager::send(const Enqueued &entry) {
+	const auto id = entry.id;
+	const auto url = entry.url;
+	_sent.emplace(id, Sent{ url, send(id, url), entry.stream });
+}
+
+void WebLoadManager::removeSent(int id) {
+	if (const auto i = _sent.find(id); i != end(_sent)) {
+		deleteDeferred(i->second.reply);
+		_sent.erase(i);
+		checkSendNext();
+	}
+}
+
+not_null<QNetworkReply*> WebLoadManager::send(int id, const QString &url) {
+	const auto result = _network->get(QNetworkRequest(url));
+	const auto handleProgress = [=](qint64 ready, qint64 total) {
+		progress(id, result, ready, total);
+	};
+	const auto handleError = [=](QNetworkReply::NetworkError error) {
+		failed(id, result, error);
+	};
+	QObject::connect(
+		result,
+		&QNetworkReply::downloadProgress,
+		handleProgress);
+	QObject::connect(result, &QNetworkReply::errorOccurred, handleError);
+	return result;
+}
+
+WebLoadManager::Sent *WebLoadManager::findSent(
+		int id,
+		not_null<QNetworkReply*> reply) {
+	const auto i = _sent.find(id);
+	return (i != end(_sent) && i->second.reply == reply)
+		? &i->second
+		: nullptr;
+}
+
+void WebLoadManager::progress(
+		int id,
+		not_null<QNetworkReply*> reply,
+		int64 ready,
+		int64 total) {
+	if (total <= 0) {
+		const auto originalContentLength = reply->attribute(
+			QNetworkRequest::OriginalContentLengthAttribute);
+		if (originalContentLength.isValid()) {
+			total = originalContentLength.toLongLong();
+		}
+	}
+	const auto statusCode = reply->attribute(
+		QNetworkRequest::HttpStatusCodeAttribute);
+	const auto status = statusCode.isValid() ? statusCode.toInt() : 200;
+	if (status == 301 || status == 302) {
+		redirect(id, reply);
+	} else if (status != 200 && status != 206 && status != 416) {
+		LOG(("Network Error: "
+			"Bad HTTP status received in WebLoadManager::onProgress() %1"
+			).arg(status));
+		failed(id, reply);
+	} else {
+		notify(id, reply, ready, std::max(ready, total));
+	}
+}
+
+void WebLoadManager::redirect(int id, not_null<QNetworkReply*> reply) {
+	const auto header = reply->header(QNetworkRequest::LocationHeader);
+	const auto url = header.toString();
+	if (url.isEmpty()) {
+		return;
+	}
+
+	if (const auto sent = findSent(id, reply)) {
+		if (!sent->redirectsLeft--) {
+			LOG(("Network Error: "
+				"Too many HTTP redirects in onFinished() "
+				"for web file loader: %1").arg(url));
+			failed(id, reply);
+			return;
+		}
+		deleteDeferred(reply);
+		sent->url = url;
+		sent->reply = send(id, url);
+	}
+}
+
+void WebLoadManager::notify(
+		int id,
+		not_null<QNetworkReply*> reply,
+		int64 ready,
+		int64 total) {
+	if (const auto sent = findSent(id, reply)) {
+		sent->ready = ready;
+		sent->total = std::max(total, int64(0));
+		if (total <= 0) {
+			LOG(("Network Error: "
+				"Bad size received for HTTP download progress "
+				"in WebLoadManager::onProgress(): %1 / %2 (bytes %3)"
+				).arg(ready
+				).arg(total
+				).arg(sent->data.size()));
+			failed(id, reply);
+			return;
+		}
+		auto bytes = reply->readAll();
+		if (sent->stream) {
+			if (total > kMaxWebFile) {
+				LOG(("Network Error: "
+					"Bad size received for HTTP download progress "
+					"in WebLoadManager::onProgress(): %1 / %2"
+					).arg(ready
+					).arg(total));
+				failed(id, reply);
+			} else {
+				queueProgressUpdate(
+					id,
+					sent->ready,
+					sent->total,
+					std::move(bytes));
+				if (ready >= total) {
+					finished(id, reply);
+				}
+			}
+		} else {
+			sent->data.append(std::move(bytes));
+			if (total > Storage::kMaxFileInMemory
+				|| sent->data.size() > Storage::kMaxFileInMemory) {
+				LOG(("Network Error: "
+					"Bad size received for HTTP download progress "
+					"in WebLoadManager::onProgress(): %1 / %2 (bytes %3)"
+					).arg(ready
+					).arg(total
+					).arg(sent->data.size()));
+				failed(id, reply);
+			} else if (ready >= total) {
+				finished(id, reply);
+			} else {
+				queueProgressUpdate(id, sent->ready, sent->total, {});
+			}
+		}
+	}
+}
+
+void WebLoadManager::failed(
+		int id,
+		not_null<QNetworkReply*> reply,
+		QNetworkReply::NetworkError error) {
+	if (const auto sent = findSent(id, reply)) {
+		LOG(("Network Error: "
+			"Failed to request '%1', error %2 (%3)"
+			).arg(sent->url
+			).arg(int(error)
+			).arg(reply->errorString()));
+		failed(id, reply);
+	}
+}
+
+void WebLoadManager::failed(int id, not_null<QNetworkReply*> reply) {
+	if ([[maybe_unused]] const auto sent = findSent(id, reply)) {
+		removeSent(id);
+		queueFailedUpdate(id);
+	}
+}
+
+void WebLoadManager::deleteDeferred(not_null<QNetworkReply*> reply) {
+	reply->deleteLater();
+	_repliesBeingDeleted.erase(
+		ranges::remove(_repliesBeingDeleted, nullptr),
+		end(_repliesBeingDeleted));
+	_repliesBeingDeleted.emplace_back(reply.get());
+}
+
+void WebLoadManager::finished(int id, not_null<QNetworkReply*> reply) {
+	if (const auto sent = findSent(id, reply)) {
+		const auto data = base::take(sent->data);
+		removeSent(id);
+		queueFinishedUpdate(id, data);
+	}
+}
+
+void WebLoadManager::clear() {
+	for (const auto &[id, sent] : base::take(_sent)) {
+		sent.reply->abort();
+		delete sent.reply;
+	}
+	for (const auto &reply : base::take(_repliesBeingDeleted)) {
+		if (reply) {
+			delete reply;
+		}
+	}
+}
+
+void WebLoadManager::queueProgressUpdate(
+		int id,
+		int64 ready,
+		int64 total,
+		QByteArray streamed) {
+	crl::on_main(this, [=, bytes = std::move(streamed)]() mutable {
+		sendUpdate(id, Progress{ ready, total, std::move(bytes) });
+	});
+}
+
+void WebLoadManager::queueFailedUpdate(int id) {
+	crl::on_main(this, [=] {
+		sendUpdate(id, Error{});
+	});
+}
+
+void WebLoadManager::queueFinishedUpdate(int id, const QByteArray &data) {
+	crl::on_main(this, [=] {
+		for (const auto &[loader, loaderId] : _ids) {
+			if (loaderId == id) {
+				break;
+			}
+		}
+		sendUpdate(id, QByteArray(data));
+	});
+}
+
+void WebLoadManager::sendUpdate(int id, Update &&data) {
+	for (const auto &[loader, loaderId] : _ids) {
+		if (loaderId == id) {
+			_updates.fire(UpdateForLoader{ loader, std::move(data) });
+			return;
+		}
+	}
+}
+
+webFileLoader::webFileLoader(
+	not_null<Main::Session*> session,
+	const QString &url,
+	const QString &to,
+	LoadFromCloudSetting fromCloud,
+	bool autoLoading,
+	uint8 cacheTag)
+: FileLoader(
+	session,
+	QString(),
+	0,
+	0,
+	UnknownFileLocation,
+	LoadToCacheAsWell,
+	fromCloud,
+	autoLoading,
+	cacheTag)
+, _url(url) {
+}
+
+webFileLoader::webFileLoader(
+	not_null<Main::Session*> session,
+	const QString &url,
+	const QString &path,
+	WebRequestType type)
+: FileLoader(
+	session,
+	path,
+	0,
+	0,
+	UnknownFileLocation,
+	LoadToFileOnly,
+	LoadFromCloudOrLocal,
+	false,
+	0)
+, _url(url)
+, _requestType(type) {
+}
+
+webFileLoader::~webFileLoader() {
+	if (!_finished) {
+		cancel();
+	}
+}
+
+QString webFileLoader::url() const {
+	return _url;
+}
+
+WebRequestType webFileLoader::requestType() const {
+	return _requestType;
+}
+
+bool webFileLoader::streamLoading() const {
+	return (_toCache == LoadToFileOnly);
+}
+
+void webFileLoader::startLoading() {
+	if (_finished) {
+		return;
+	} else if (!_manager) {
+		_manager = GetManager();
+		_manager->updates(
+			this
+		) | rpl::on_next([=](const Update &data) {
+			if (const auto progress = std::get_if<Progress>(&data)) {
+				loadProgress(
+					progress->ready,
+					progress->total,
+					progress->streamed);
+			} else if (const auto bytes = std::get_if<QByteArray>(&data)) {
+				loadFinished(*bytes);
+			} else {
+				loadFailed();
+			}
+		}, _managerLifetime);
+	}
+	_manager->enqueue(this);
+}
+
+int64 webFileLoader::currentOffset() const {
+	return _ready;
+}
+
+void webFileLoader::loadProgress(
+		qint64 ready,
+		qint64 total,
+		const QByteArray &streamed) {
+	_fullSize = _loadSize = total;
+	_ready = ready;
+	if (!streamed.isEmpty()
+		&& !writeResultPart(_streamedOffset, bytes::make_span(streamed))) {
+		loadFailed();
+	} else {
+		_streamedOffset += streamed.size();
+		notifyAboutProgress();
+	}
+}
+
+void webFileLoader::loadFinished(const QByteArray &data) {
+	cancelRequest();
+	if (writeResultPart(0, bytes::make_span(data))) {
+		finalizeResult();
+	}
+}
+
+void webFileLoader::loadFailed() {
+	cancel(FailureReason::OtherFailure);
+}
+
+Storage::Cache::Key webFileLoader::cacheKey() const {
+	return Data::UrlCacheKey(_url);
+}
+
+std::optional<MediaKey> webFileLoader::fileLocationKey() const {
+	return std::nullopt;
+}
+
+void webFileLoader::cancelHook() {
+	cancelRequest();
+}
+
+void webFileLoader::cancelRequest() {
+	if (!_manager) {
+		return;
+	}
+	_managerLifetime.destroy();
+	_manager->remove(this);
+	_manager = nullptr;
+}

Telegram/SourceFiles/storage/file_download_web.h

@@ -0,0 +1,63 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#pragma once
+
+#include "storage/file_download.h"
+
+class WebLoadManager;
+
+enum class WebRequestType {
+	FullLoad,
+	OnlySize,
+};
+
+class webFileLoader final : public FileLoader {
+public:
+	webFileLoader(
+		not_null<Main::Session*> session,
+		const QString &url,
+		const QString &to,
+		LoadFromCloudSetting fromCloud,
+		bool autoLoading,
+		uint8 cacheTag);
+	webFileLoader(
+		not_null<Main::Session*> session,
+		const QString &url,
+		const QString &path,
+		WebRequestType type);
+	~webFileLoader();
+
+	[[nodiscard]] QString url() const;
+	[[nodiscard]] WebRequestType requestType() const;
+	[[nodiscard]] bool streamLoading() const;
+
+	int64 currentOffset() const override;
+
+private:
+	void cancelRequest();
+	void cancelHook() override;
+	void startLoading() override;
+	Storage::Cache::Key cacheKey() const override;
+	std::optional<MediaKey> fileLocationKey() const override;
+
+	void loadProgress(
+		qint64 ready,
+		qint64 size,
+		const QByteArray &streamed);
+	void loadFinished(const QByteArray &data);
+	void loadFailed();
+
+	const QString _url;
+	int64 _ready = 0;
+	int64 _streamedOffset = 0;
+	WebRequestType _requestType = {};
+
+	std::shared_ptr<WebLoadManager> _manager;
+	rpl::lifetime _managerLifetime;
+
+};

Telegram/SourceFiles/storage/file_upload.h

@@ -0,0 +1,193 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#pragma once
+
+#include "api/api_common.h"
+#include "base/timer.h"
+#include "base/weak_ptr.h"
+#include "mtproto/facade.h"
+
+class ApiWrap;
+struct FilePrepareResult;
+
+namespace Api {
+enum class SendProgressType;
+} // namespace Api
+
+namespace Main {
+class Session;
+} // namespace Main
+
+namespace Storage {
+
+// MTP big files methods used for files greater than 30mb.
+constexpr auto kUseBigFilesFrom = 30 * 1024 * 1024;
+
+struct UploadedMedia {
+	uint64 id = 0;
+	FullMsgId fullId;
+	Api::RemoteFileInfo info;
+	Api::SendOptions options;
+	bool edit = false;
+};
+
+struct UploadSecureProgress {
+	FullMsgId fullId;
+	int64 offset = 0;
+	int64 size = 0;
+};
+
+struct UploadSecureDone {
+	FullMsgId fullId;
+	uint64 fileId = 0;
+	int partsCount = 0;
+};
+
+class Uploader final : public base::has_weak_ptr {
+public:
+	explicit Uploader(not_null<ApiWrap*> api);
+	~Uploader();
+
+	[[nodiscard]] Main::Session &session() const;
+	[[nodiscard]] FullMsgId currentUploadId() const;
+
+	void upload(
+		FullMsgId itemId,
+		const std::shared_ptr<FilePrepareResult> &file);
+
+	void pause(FullMsgId itemId);
+	void cancel(FullMsgId itemId);
+	void cancelAll();
+
+	[[nodiscard]] rpl::producer<UploadedMedia> photoReady() const {
+		return _photoReady.events();
+	}
+	[[nodiscard]] rpl::producer<UploadedMedia> documentReady() const {
+		return _documentReady.events();
+	}
+	[[nodiscard]] rpl::producer<UploadSecureDone> secureReady() const {
+		return _secureReady.events();
+	}
+	[[nodiscard]] rpl::producer<FullMsgId> photoProgress() const {
+		return _photoProgress.events();
+	}
+	[[nodiscard]] rpl::producer<FullMsgId> documentProgress() const {
+		return _documentProgress.events();
+	}
+	[[nodiscard]] auto secureProgress() const
+	-> rpl::producer<UploadSecureProgress> {
+		return _secureProgress.events();
+	}
+	[[nodiscard]] rpl::producer<FullMsgId> photoFailed() const {
+		return _photoFailed.events();
+	}
+	[[nodiscard]] rpl::producer<FullMsgId> documentFailed() const {
+		return _documentFailed.events();
+	}
+	[[nodiscard]] rpl::producer<FullMsgId> secureFailed() const {
+		return _secureFailed.events();
+	}
+
+	[[nodiscard]] rpl::producer<FullMsgId> nonPremiumDelays() const {
+		return _nonPremiumDelays.events();
+	}
+
+	void unpause();
+	void stopSessions();
+
+private:
+	struct Entry;
+	struct Request;
+
+	enum class SendResult : uchar {
+		Success,
+		Failed,
+		DcIndexFull,
+	};
+
+	void maybeSend();
+	[[nodiscard]] bool canAddDcIndex() const;
+	[[nodiscard]] std::optional<uchar> chooseDcIndexForNextRequest(
+		const base::flat_set<uchar> &used);
+	[[nodiscard]] Entry *chooseEntryForNextRequest();
+	[[nodiscard]] SendResult sendPart(not_null<Entry*> entry, uchar dcIndex);
+	[[nodiscard]] auto sendPendingPart(not_null<Entry*> entry, uchar dcIndex)
+		-> SendResult;
+	[[nodiscard]] auto sendDocPart(not_null<Entry*> entry, uchar dcIndex)
+		-> SendResult;
+	[[nodiscard]] auto sendSlicedPart(not_null<Entry*> entry, uchar dcIndex)
+		-> SendResult;
+	[[nodiscard]] QByteArray readDocPart(not_null<Entry*> entry);
+	void removeDcIndex();
+
+	template <typename Prepared>
+	void sendPreparedRequest(Prepared &&prepared, Request &&request);
+
+	void maybeFinishFront();
+	void finishFront();
+
+	void partLoaded(const MTPBool &result, mtpRequestId requestId);
+	void partFailed(const MTP::Error &error, mtpRequestId requestId);
+	Request finishRequest(mtpRequestId requestId);
+
+	void uploadVideoCover(
+		UploadedMedia &&video,
+		std::shared_ptr<FilePrepareResult> videoCover);
+	void uploadCoverAsPhoto(FullMsgId videoId, UploadedMedia &&cover);
+
+	void processPhotoProgress(FullMsgId itemId);
+	void processPhotoFailed(FullMsgId itemId);
+	void processDocumentProgress(FullMsgId itemId);
+	void processDocumentFailed(FullMsgId itemId);
+
+	void notifyFailed(const Entry &entry);
+	void failed(FullMsgId itemId);
+	void cancelRequests(FullMsgId itemId);
+	void cancelAllRequests();
+	void clear();
+
+	void sendProgressUpdate(
+		not_null<HistoryItem*> item,
+		Api::SendProgressType type,
+		int progress = 0);
+
+	const not_null<ApiWrap*> _api;
+
+	std::vector<Entry> _queue;
+
+	base::flat_map<mtpRequestId, Request> _requests;
+	std::vector<int> _sentPerDcIndex;
+
+	// Fast requests since the latest dc index addition.
+	base::flat_set<uchar> _dcIndicesWithFastRequests;
+	crl::time _latestDcIndexAdded = 0;
+	crl::time _latestDcIndexRemoved = 0;
+	std::vector<Request> _pendingFromRemovedDcIndices;
+
+	base::flat_map<FullMsgId, FullMsgId> _videoIdToCoverId;
+	base::flat_map<FullMsgId, UploadedMedia> _videoWaitingCover;
+
+	FullMsgId _pausedId;
+	base::Timer _nextTimer, _stopSessionsTimer;
+
+	rpl::event_stream<UploadedMedia> _photoReady;
+	rpl::event_stream<UploadedMedia> _documentReady;
+	rpl::event_stream<UploadSecureDone> _secureReady;
+	rpl::event_stream<FullMsgId> _photoProgress;
+	rpl::event_stream<FullMsgId> _documentProgress;
+	rpl::event_stream<UploadSecureProgress> _secureProgress;
+	rpl::event_stream<FullMsgId> _photoFailed;
+	rpl::event_stream<FullMsgId> _documentFailed;
+	rpl::event_stream<FullMsgId> _secureFailed;
+	rpl::event_stream<FullMsgId> _nonPremiumDelays;
+
+	rpl::lifetime _lifetime;
+
+};
+
+} // namespace Storage

Telegram/SourceFiles/storage/localimageloader.h

@@ -0,0 +1,298 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#pragma once
+
+#include "base/variant.h"
+#include "api/api_common.h"
+
+namespace Ui {
+struct PreparedFileInformation;
+} // namespace Ui
+
+namespace Main {
+class Session;
+} // namespace Main
+
+// Load files up to 2'000 MB.
+constexpr auto kFileSizeLimit = 2'000 * int64(1024 * 1024);
+
+// Load files up to 4'000 MB.
+constexpr auto kFileSizePremiumLimit = 4'000 * int64(1024 * 1024);
+
+extern const char kOptionSendLargePhotos[];
+
+[[nodiscard]] int PhotoSideLimit();
+
+enum class SendMediaType {
+	Photo,
+	Audio,
+	Round,
+	File,
+	ThemeFile,
+	Secure,
+};
+
+using TaskId = void*; // no interface, just id
+inline constexpr auto kEmptyTaskId = TaskId();
+
+class Task {
+public:
+	virtual void process() = 0; // is executed in a separate thread
+	virtual void finish() = 0; // is executed in the same as TaskQueue thread
+	virtual ~Task() = default;
+
+	TaskId id() const {
+		return static_cast<TaskId>(const_cast<Task*>(this));
+	}
+
+};
+
+class TaskQueueWorker;
+class TaskQueue : public QObject {
+	Q_OBJECT
+
+public:
+	explicit TaskQueue(crl::time stopTimeoutMs = 0); // <= 0 - never stop worker
+
+	TaskId addTask(std::unique_ptr<Task> &&task);
+	void addTasks(std::vector<std::unique_ptr<Task>> &&tasks);
+	void cancelTask(TaskId id); // this task finish() won't be called
+
+	~TaskQueue();
+
+Q_SIGNALS:
+	void taskAdded();
+
+public Q_SLOTS:
+	void onTaskProcessed();
+	void stop();
+
+private:
+	friend class TaskQueueWorker;
+
+	void wakeThread();
+
+	std::deque<std::unique_ptr<Task>> _tasksToProcess;
+	std::deque<std::unique_ptr<Task>> _tasksToFinish;
+	TaskId _taskInProcessId = TaskId();
+	QMutex _tasksToProcessMutex, _tasksToFinishMutex;
+	QThread *_thread = nullptr;
+	TaskQueueWorker *_worker = nullptr;
+	QTimer *_stopTimer = nullptr;
+
+};
+
+class TaskQueueWorker : public QObject {
+	Q_OBJECT
+
+public:
+	TaskQueueWorker(TaskQueue *queue) : _queue(queue) {
+	}
+
+Q_SIGNALS:
+	void taskProcessed();
+
+public Q_SLOTS:
+	void onTaskAdded();
+
+private:
+	TaskQueue *_queue;
+	bool _inTaskAdded = false;
+
+};
+
+struct SendingAlbum {
+	struct Item {
+		explicit Item(TaskId taskId);
+
+		TaskId taskId = kEmptyTaskId;
+		uint64 randomId = 0;
+		FullMsgId msgId;
+		std::optional<MTPInputSingleMedia> media;
+	};
+
+	SendingAlbum();
+
+	void fillMedia(
+		not_null<HistoryItem*> item,
+		const MTPInputMedia &media,
+		uint64 randomId);
+	void refreshMediaCaption(not_null<HistoryItem*> item);
+	void removeItem(not_null<HistoryItem*> item);
+
+	uint64 groupId = 0;
+	std::vector<Item> items;
+	Api::SendOptions options;
+	bool sent = false;
+
+};
+
+struct FileLoadTo {
+	FileLoadTo(
+		PeerId peer,
+		Api::SendOptions options,
+		FullReplyTo replyTo,
+		MsgId replaceMediaOf)
+	: peer(peer)
+	, options(options)
+	, replyTo(replyTo)
+	, replaceMediaOf(replaceMediaOf) {
+	}
+	PeerId peer;
+	Api::SendOptions options;
+	FullReplyTo replyTo;
+	MsgId replaceMediaOf;
+};
+
+using UploadFileParts = std::vector<QByteArray>;
+struct FilePrepareDescriptor {
+	TaskId taskId = kEmptyTaskId;
+	base::required<uint64> id;
+	SendMediaType type = SendMediaType::File;
+	FileLoadTo to = { PeerId(), Api::SendOptions(), FullReplyTo(), MsgId() };
+	TextWithTags caption;
+	bool spoiler = false;
+	std::shared_ptr<SendingAlbum> album;
+};
+struct FilePrepareResult {
+	explicit FilePrepareResult(FilePrepareDescriptor &&descriptor);
+
+	TaskId taskId = kEmptyTaskId;
+	uint64 id = 0;
+	FileLoadTo to;
+	std::shared_ptr<SendingAlbum> album;
+	SendMediaType type = SendMediaType::File;
+	QString filepath;
+	QByteArray content;
+
+	QString filename;
+	QString filemime;
+	int64 filesize = 0;
+	UploadFileParts fileparts;
+	QByteArray filemd5;
+	int64 partssize = 0;
+
+	uint64 thumbId = 0; // id is always file-id of media, thumbId is file-id of thumb ( == id for photos)
+	QString thumbname;
+	UploadFileParts thumbparts;
+	QByteArray thumbbytes;
+	QByteArray thumbmd5;
+	QImage thumb;
+
+	QImage goodThumbnail;
+	QByteArray goodThumbnailBytes;
+
+	MTPPhoto photo = MTP_photoEmpty(MTP_long(0));
+	MTPDocument document = MTP_documentEmpty(MTP_long(0));
+
+	PreparedPhotoThumbs photoThumbs;
+	TextWithTags caption;
+	bool spoiler = false;
+
+	std::vector<MTPInputDocument> attachedStickers;
+
+	std::shared_ptr<FilePrepareResult> videoCover;
+
+	void setFileData(const QByteArray &filedata);
+	void setThumbData(const QByteArray &thumbdata);
+
+};
+
+[[nodiscard]] std::shared_ptr<FilePrepareResult> MakePreparedFile(
+	FilePrepareDescriptor &&descriptor);
+
+class FileLoadTask final : public Task {
+public:
+	static std::unique_ptr<Ui::PreparedFileInformation> ReadMediaInformation(
+		const QString &filepath,
+		const QByteArray &content,
+		const QString &filemime);
+	static bool FillImageInformation(
+		QImage &&image,
+		bool animated,
+		std::unique_ptr<Ui::PreparedFileInformation> &result,
+		QByteArray content = {},
+		QByteArray format = {});
+
+	FileLoadTask(
+		not_null<Main::Session*> session,
+		const QString &filepath,
+		const QByteArray &content,
+		std::unique_ptr<Ui::PreparedFileInformation> information,
+		std::unique_ptr<FileLoadTask> videoCover,
+		SendMediaType type,
+		const FileLoadTo &to,
+		const TextWithTags &caption,
+		bool spoiler,
+		std::shared_ptr<SendingAlbum> album = nullptr,
+		uint64 idOverride = 0);
+	FileLoadTask(
+		not_null<Main::Session*> session,
+		const QByteArray &voice,
+		crl::time duration,
+		const VoiceWaveform &waveform,
+		bool video,
+		const FileLoadTo &to,
+		const TextWithTags &caption);
+	~FileLoadTask();
+
+	uint64 fileid() const {
+		return _id;
+	}
+
+	struct Args {
+		bool generateGoodThumbnail = true;
+	};
+	void process(Args &&args);
+
+	void process() override {
+		process({});
+	}
+	void finish() override;
+
+	[[nodiscard]] auto peekResult() const
+		-> const std::shared_ptr<FilePrepareResult> &;
+
+private:
+	static bool CheckForSong(
+		const QString &filepath,
+		const QByteArray &content,
+		std::unique_ptr<Ui::PreparedFileInformation> &result);
+	static bool CheckForVideo(
+		const QString &filepath,
+		const QByteArray &content,
+		std::unique_ptr<Ui::PreparedFileInformation> &result);
+	static bool CheckForImage(
+		const QString &filepath,
+		const QByteArray &content,
+		std::unique_ptr<Ui::PreparedFileInformation> &result);
+
+	template <typename Mimes, typename Extensions>
+	static bool CheckMimeOrExtensions(const QString &filepath, const QString &filemime, Mimes &mimes, Extensions &extensions);
+
+	std::unique_ptr<Ui::PreparedFileInformation> readMediaInformation(const QString &filemime) const;
+	void removeFromAlbum();
+
+	uint64 _id = 0;
+	base::weak_ptr<Main::Session> _session;
+	MTP::DcId _dcId = 0;
+	FileLoadTo _to;
+	const std::shared_ptr<SendingAlbum> _album;
+	QString _filepath;
+	QByteArray _content;
+	std::unique_ptr<FileLoadTask> _videoCover;
+	std::unique_ptr<Ui::PreparedFileInformation> _information;
+	crl::time _duration = 0;
+	VoiceWaveform _waveform;
+	SendMediaType _type;
+	TextWithTags _caption;
+	bool _spoiler = false;
+
+	std::shared_ptr<FilePrepareResult> _result;
+
+};

Telegram/SourceFiles/storage/localstorage.h

@@ -0,0 +1,97 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#pragma once
+
+#include "storage/file_download.h"
+#include "storage/localimageloader.h"
+
+#include <QtCore/QTimer>
+
+class History;
+
+namespace Data {
+class WallPaper;
+class DocumentMedia;
+} // namespace Data
+
+namespace Lang {
+struct Language;
+} // namespace Lang
+
+namespace Storage {
+namespace details {
+struct ReadSettingsContext;
+} // namespace details
+class EncryptionKey;
+} // namespace Storage
+
+namespace Window {
+namespace Theme {
+struct Object;
+struct Saved;
+} // namespace Theme
+} // namespace Window
+
+namespace Export {
+struct Settings;
+} // namespace Export
+
+namespace MTP {
+class AuthKey;
+using AuthKeyPtr = std::shared_ptr<AuthKey>;
+} // namespace MTP
+
+namespace Local {
+
+void start();
+void sync();
+void finish();
+
+void writeSettings();
+void rewriteSettingsIfNeeded();
+
+void writeAutoupdatePrefix(const QString &prefix);
+QString readAutoupdatePrefix();
+
+void writeBackground(const Data::WallPaper &paper, const QImage &image);
+bool readBackground();
+void moveLegacyBackground(
+	const QString &fromBasePath,
+	const MTP::AuthKeyPtr &fromLocalKey,
+	uint64 legacyBackgroundKeyDay,
+	uint64 legacyBackgroundKeyNight);
+
+void reset();
+
+int32 oldSettingsVersion();
+
+void countVoiceWaveform(not_null<Data::DocumentMedia*> media);
+
+void cancelTask(TaskId id);
+
+void writeTheme(const Window::Theme::Saved &saved);
+void clearTheme();
+[[nodiscard]] Window::Theme::Saved readThemeAfterSwitch();
+
+[[nodiscard]] Window::Theme::Object ReadThemeContent();
+
+void writeLangPack();
+void pushRecentLanguage(const Lang::Language &language);
+std::vector<Lang::Language> readRecentLanguages();
+void saveRecentLanguages(const std::vector<Lang::Language> &list);
+void removeRecentLanguage(const QString &id);
+void incrementRecentHashtag(RecentHashtagPack &recent, const QString &tag);
+
+bool readOldMtpData(
+	bool remove,
+	Storage::details::ReadSettingsContext &context);
+bool readOldUserSettings(
+	bool remove,
+	Storage::details::ReadSettingsContext &context);
+
+} // namespace Local

Telegram/SourceFiles/storage/serialize_common.cpp

@@ -0,0 +1,48 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#include "storage/serialize_common.h"
+
+namespace Serialize {
+
+ByteArrayWriter::ByteArrayWriter(int expectedSize)
+: _stream(&_result, QIODevice::WriteOnly) {
+	if (expectedSize) {
+		_result.reserve(expectedSize);
+	}
+	_stream.setVersion(QDataStream::Qt_5_1);
+}
+
+QByteArray ByteArrayWriter::result() && {
+	_stream.device()->close();
+	return std::move(_result);
+}
+
+ByteArrayReader::ByteArrayReader(QByteArray data)
+: _data(std::move(data))
+, _stream(&_data, QIODevice::ReadOnly) {
+	_stream.setVersion(QDataStream::Qt_5_1);
+}
+
+void writeColor(QDataStream &stream, const QColor &color) {
+	stream << (quint32(uchar(color.red()))
+		| (quint32(uchar(color.green())) << 8)
+		| (quint32(uchar(color.blue())) << 16)
+		| (quint32(uchar(color.alpha())) << 24));
+}
+
+QColor readColor(QDataStream &stream) {
+	auto value = quint32();
+	stream >> value;
+	return QColor(
+		int(value & 0xFFU),
+		int((value >> 8) & 0xFFU),
+		int((value >> 16) & 0xFFU),
+		int((value >> 24) & 0xFFU));
+}
+
+} // namespace Serialize

Telegram/SourceFiles/storage/serialize_common.h

@@ -0,0 +1,175 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#pragma once
+
+#include "mtproto/mtproto_auth_key.h"
+#include "base/bytes.h"
+
+#include <QtCore/QDataStream>
+
+namespace Serialize {
+
+class ByteArrayWriter final {
+public:
+	explicit ByteArrayWriter(int expectedSize = 0);
+
+	[[nodiscard]] QDataStream &underlying() {
+		return _stream;
+	}
+	[[nodiscard]] operator QDataStream &() {
+		return _stream;
+	}
+	[[nodiscard]] QByteArray result() &&;
+
+private:
+	QByteArray _result;
+	QDataStream _stream;
+
+};
+
+template <typename T>
+inline ByteArrayWriter &operator<<(ByteArrayWriter &stream, const T &data) {
+	stream.underlying() << data;
+	return stream;
+}
+
+class ByteArrayReader final {
+public:
+	explicit ByteArrayReader(QByteArray data);
+
+	[[nodiscard]] QDataStream &underlying() {
+		return _stream;
+	}
+	[[nodiscard]] operator QDataStream &() {
+		return _stream;
+	}
+
+	[[nodiscard]] bool atEnd() const {
+		return _stream.atEnd();
+	}
+	[[nodiscard]] bool status() const {
+		return _stream.status();
+	}
+	[[nodiscard]] bool ok() const {
+		return _stream.status() == QDataStream::Ok;
+	}
+
+private:
+	QByteArray _data;
+	QDataStream _stream;
+
+};
+
+template <typename T>
+inline ByteArrayReader &operator>>(ByteArrayReader &stream, T &data) {
+	if (!stream.ok()) {
+		data = T();
+	} else {
+		stream.underlying() >> data;
+		if (!stream.ok()) {
+			data = T();
+		}
+	}
+	return stream;
+}
+
+inline int stringSize(const QString &str) {
+	return sizeof(quint32) + str.size() * sizeof(ushort);
+}
+
+inline int bytearraySize(const QByteArray &arr) {
+	return sizeof(quint32) + arr.size();
+}
+
+inline int bytesSize(bytes::const_span bytes) {
+	return sizeof(quint32) + bytes.size();
+}
+
+inline int colorSize() {
+	return sizeof(quint32);
+}
+
+void writeColor(QDataStream &stream, const QColor &color);
+QColor readColor(QDataStream &stream);
+
+struct ReadBytesVectorWrap {
+	bytes::vector &bytes;
+};
+
+inline ReadBytesVectorWrap bytes(bytes::vector &bytes) {
+	return ReadBytesVectorWrap { bytes };
+}
+
+// Compatible with QDataStream &operator>>(QDataStream &, QByteArray &);
+inline QDataStream &operator>>(QDataStream &stream, ReadBytesVectorWrap data) {
+	auto &bytes = data.bytes;
+	bytes.clear();
+	quint32 len;
+	stream >> len;
+	if (stream.status() != QDataStream::Ok || len == 0xFFFFFFFF) {
+		return stream;
+	}
+
+	constexpr auto kStep = quint32(1024 * 1024);
+	for (auto allocated = quint32(0); allocated < len;) {
+		auto blockSize = qMin(kStep, len - allocated);
+		bytes.resize(allocated + blockSize);
+		if (stream.readRawData(reinterpret_cast<char*>(bytes.data()) + allocated, blockSize) != blockSize) {
+			bytes.clear();
+			stream.setStatus(QDataStream::ReadPastEnd);
+			return stream;
+		}
+		allocated += blockSize;
+	}
+
+	return stream;
+}
+
+struct WriteBytesWrap {
+	bytes::const_span bytes;
+};
+
+inline WriteBytesWrap bytes(bytes::const_span bytes) {
+	return WriteBytesWrap { bytes };
+}
+
+inline QDataStream &operator<<(QDataStream &stream, WriteBytesWrap data) {
+	auto bytes = data.bytes;
+	if (bytes.empty()) {
+		stream << quint32(0xFFFFFFFF);
+	} else {
+		auto size = quint32(bytes.size());
+		stream << size;
+		stream.writeRawData(reinterpret_cast<const char*>(bytes.data()), size);
+	}
+	return stream;
+}
+
+inline QDataStream &operator<<(QDataStream &stream, ReadBytesVectorWrap data) {
+	return stream << WriteBytesWrap { data.bytes };
+}
+
+inline int dateTimeSize() {
+	return (sizeof(qint64) + sizeof(quint32) + sizeof(qint8));
+}
+
+template <typename T>
+inline T read(QDataStream &stream) {
+	auto result = T();
+	stream >> result;
+	return result;
+}
+
+template <>
+inline MTP::AuthKey::Data read<MTP::AuthKey::Data>(QDataStream &stream) {
+	auto result = MTP::AuthKey::Data();
+	stream.readRawData(reinterpret_cast<char*>(result.data()), result.size());
+	return result;
+}
+
+} // namespace Serialize

Telegram/SourceFiles/storage/serialize_document.cpp

@@ -0,0 +1,308 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#include "storage/serialize_document.h"
+
+#include "storage/serialize_common.h"
+#include "storage/serialize_peer.h"
+#include "data/data_session.h"
+#include "data/stickers/data_stickers.h"
+#include "ui/image/image.h"
+#include "main/main_session.h"
+
+namespace Serialize {
+namespace {
+
+constexpr auto kVersionTag = int32(0x7FFFFFFF);
+constexpr auto kVersion = 6;
+
+enum StickerSetType {
+	StickerSetTypeEmpty = 0,
+	StickerSetTypeID = 1,
+	StickerSetTypeShortName = 2,
+	StickerSetTypeEmoji = 3,
+	StickerSetTypeMasks = 4,
+};
+
+} // namespace
+
+void Document::writeToStream(QDataStream &stream, DocumentData *document) {
+	stream
+		<< quint64(document->id)
+		<< quint64(document->_access)
+		<< qint32(document->date)
+		<< document->_fileReference
+		<< qint32(kVersionTag)
+		<< qint32(kVersion)
+		<< document->filename()
+		<< document->mimeString()
+		<< qint32(document->_dc)
+		// FileSize: Right now any file size fits 32 bit.
+		<< qint32(uint32(document->size))
+		<< qint32(document->dimensions.width())
+		<< qint32(document->dimensions.height())
+		<< qint32(document->type);
+	if (const auto sticker = document->sticker()) {
+		stream << sticker->alt;
+		if (sticker->setType == Data::StickersType::Emoji) {
+			stream << qint32(StickerSetTypeEmoji);
+		} else if (sticker->setType == Data::StickersType::Masks) {
+			stream << qint32(StickerSetTypeMasks);
+		} else if (sticker->set.id) {
+			stream << qint32(StickerSetTypeID);
+		} else {
+			stream << qint32(StickerSetTypeEmpty);
+		}
+	}
+	stream << qint64(document->hasDuration() ? document->duration() : -1);
+	if (document->type == StickerDocument) {
+		const auto premium = document->isPremiumSticker()
+			|| document->isPremiumEmoji();
+		stream << qint32(premium ? 1 : 0);
+		stream << qint32(document->emojiUsesTextColor() ? 1 : 0);
+	}
+	writeImageLocation(stream, document->thumbnailLocation());
+	stream << qint32(document->thumbnailByteSize());
+	writeImageLocation(stream, document->videoThumbnailLocation());
+	stream
+		<< qint32(document->videoThumbnailByteSize())
+		<< qint32(document->inlineThumbnailIsPath() ? 1 : 0)
+		<< document->inlineThumbnailBytes();
+}
+
+DocumentData *Document::readFromStreamHelper(
+		not_null<Main::Session*> session,
+		int streamAppVersion,
+		QDataStream &stream,
+		const StickerSetInfo *info) {
+	quint64 id, access;
+	QString name, mime;
+	qint32 date, dc, size, width, height, type, versionTag, version = 0;
+	QByteArray fileReference;
+	stream >> id >> access >> date;
+	if (streamAppVersion >= 9061) {
+		if (streamAppVersion >= 1003013) {
+			stream >> fileReference;
+		}
+		stream >> versionTag;
+		if (versionTag == kVersionTag) {
+			stream >> version;
+		}
+	} else {
+		versionTag = 0;
+		version = 0;
+	}
+	stream
+		>> name
+		>> mime
+		>> dc
+		>> size // FileSize: Right now any file size fits 32 bit.
+		>> width
+		>> height
+		>> type;
+
+	QVector<MTPDocumentAttribute> attributes;
+	if (!name.isEmpty()) {
+		attributes.push_back(MTP_documentAttributeFilename(MTP_string(name)));
+	}
+
+	qint64 duration = -1;
+	qint32 isPremiumSticker = 0;
+	qint32 useTextColor = 0;
+	if (type == StickerDocument) {
+		QString alt;
+		qint32 typeOfSet;
+		stream >> alt >> typeOfSet;
+		if (version >= 6) {
+			stream >> duration >> isPremiumSticker >> useTextColor;
+		} else if (version >= 3) {
+			qint32 oldDuration = -1;
+			stream >> oldDuration;
+			duration = (oldDuration < 0) ? oldDuration : oldDuration * 1000;
+			if (version >= 4) {
+				stream >> isPremiumSticker;
+				if (version >= 5) {
+					stream >> useTextColor;
+				}
+			}
+		}
+		if (typeOfSet == StickerSetTypeEmpty) {
+			attributes.push_back(MTP_documentAttributeSticker(MTP_flags(0), MTP_string(alt), MTP_inputStickerSetEmpty(), MTPMaskCoords()));
+		} else if (info) {
+			if (info->setId == Data::Stickers::DefaultSetId
+				|| info->setId == Data::Stickers::CloudRecentSetId
+				|| info->setId == Data::Stickers::CloudRecentAttachedSetId
+				|| info->setId == Data::Stickers::FavedSetId
+				|| info->setId == Data::Stickers::CustomSetId
+				|| info->setId == Data::Stickers::CollectibleSetId) {
+				typeOfSet = StickerSetTypeEmpty;
+			}
+
+			switch (typeOfSet) {
+			case StickerSetTypeID: {
+				attributes.push_back(MTP_documentAttributeSticker(MTP_flags(0), MTP_string(alt), MTP_inputStickerSetID(MTP_long(info->setId), MTP_long(info->accessHash)), MTPMaskCoords()));
+			} break;
+			case StickerSetTypeMasks: {
+				attributes.push_back(MTP_documentAttributeSticker(MTP_flags(MTPDdocumentAttributeSticker::Flag::f_mask), MTP_string(alt), MTP_inputStickerSetID(MTP_long(info->setId), MTP_long(info->accessHash)), MTPMaskCoords()));
+			} break;
+			case StickerSetTypeEmoji: {
+				if (version < 5) {
+					// We didn't store useTextColor yet, can't use.
+					stream.setStatus(QDataStream::ReadCorruptData);
+					return nullptr;
+				}
+				using Flag = MTPDdocumentAttributeCustomEmoji::Flag;
+				attributes.push_back(MTP_documentAttributeCustomEmoji(
+					MTP_flags((isPremiumSticker ? Flag(0) : Flag::f_free)
+						| (useTextColor ? Flag::f_text_color : Flag(0))),
+					MTP_string(alt),
+					MTP_inputStickerSetID(
+						MTP_long(info->setId),
+						MTP_long(info->accessHash))));
+			} break;
+			case StickerSetTypeEmpty:
+			default: {
+				attributes.push_back(MTP_documentAttributeSticker(MTP_flags(0), MTP_string(alt), MTP_inputStickerSetEmpty(), MTPMaskCoords()));
+			} break;
+			}
+		}
+	} else {
+		if (version >= 6) {
+			stream >> duration;
+		} else {
+			qint32 oldDuration = -1;
+			stream >> oldDuration;
+			duration = (oldDuration < 0) ? oldDuration : oldDuration * 1000;
+		}
+		if (type == AnimatedDocument) {
+			attributes.push_back(MTP_documentAttributeAnimated());
+		}
+	}
+	std::optional<ImageLocation> videoThumb;
+	qint32 thumbnailByteSize = 0, videoThumbnailByteSize = 0;
+	qint32 inlineThumbnailIsPath = 0;
+	QByteArray inlineThumbnailBytes;
+	const auto thumb = readImageLocation(streamAppVersion, stream);
+	if (version >= 1) {
+		stream >> thumbnailByteSize;
+		videoThumb = readImageLocation(streamAppVersion, stream);
+		stream >> videoThumbnailByteSize;
+		if (version >= 2) {
+			stream >> inlineThumbnailIsPath >> inlineThumbnailBytes;
+		}
+	} else {
+		videoThumb = ImageLocation();
+	}
+	if (width > 0 && height > 0) {
+		if (duration >= 0) {
+			auto flags = MTPDdocumentAttributeVideo::Flags(0);
+			if (type == RoundVideoDocument) {
+				flags |= MTPDdocumentAttributeVideo::Flag::f_round_message;
+			}
+			attributes.push_back(MTP_documentAttributeVideo(
+				MTP_flags(flags),
+				MTP_double(duration / 1000.),
+				MTP_int(width),
+				MTP_int(height),
+				MTPint(), // preload_prefix_size
+				MTPdouble(), // video_start_ts
+				MTPstring())); // video_codec
+		} else {
+			attributes.push_back(MTP_documentAttributeImageSize(
+				MTP_int(width),
+				MTP_int(height)));
+		}
+	}
+
+	if ((stream.status() != QDataStream::Ok)
+		|| (!dc && !access)
+		|| !thumb
+		|| !videoThumb) {
+		stream.setStatus(QDataStream::ReadCorruptData);
+		return nullptr;
+	}
+	const auto storage = std::get_if<StorageFileLocation>(
+		&thumb->file().data);
+	if (thumb->valid()
+		&& (!storage || !storage->isDocumentThumbnail())) {
+		stream.setStatus(QDataStream::ReadCorruptData);
+		// We can't convert legacy thumbnail location to modern, because
+		// size letter ('s' or 'm') is lost, it was not saved in legacy.
+		return nullptr;
+	}
+	return session->data().document(
+		id,
+		access,
+		fileReference,
+		date,
+		attributes,
+		mime,
+		InlineImageLocation{
+			inlineThumbnailBytes,
+			(inlineThumbnailIsPath == 1),
+		},
+		ImageWithLocation{
+			.location = *thumb,
+			.bytesCount = thumbnailByteSize
+		},
+		ImageWithLocation{
+			.location = *videoThumb,
+			.bytesCount = videoThumbnailByteSize
+		},
+		(isPremiumSticker == 1),
+		dc,
+		int64(uint32(size)));
+}
+
+DocumentData *Document::readStickerFromStream(
+		not_null<Main::Session*> session,
+		int streamAppVersion,
+		QDataStream &stream,
+		const StickerSetInfo &info) {
+	return readFromStreamHelper(session, streamAppVersion, stream, &info);
+}
+
+DocumentData *Document::readFromStream(
+		not_null<Main::Session*> session,
+		int streamAppVersion,
+		QDataStream &stream) {
+	return readFromStreamHelper(session, streamAppVersion, stream, nullptr);
+}
+
+int Document::sizeInStream(DocumentData *document) {
+	int result = 0;
+
+	// id + access + date
+	result += sizeof(quint64) + sizeof(quint64) + sizeof(qint32);
+	// file_reference + version tag + version
+	result += bytearraySize(document->_fileReference) + sizeof(qint32) * 2;
+	// + namelen + name + mimelen + mime + dc + size
+	result += stringSize(document->filename()) + stringSize(document->mimeString()) + sizeof(qint32) + sizeof(qint32);
+	// + width + height
+	result += sizeof(qint32) + sizeof(qint32);
+	// + type
+	result += sizeof(qint32);
+
+	if (auto sticker = document->sticker()) { // type == StickerDocument
+		// + altlen + alt + type-of-set
+		result += stringSize(sticker->alt) + sizeof(qint32);
+	} else {
+		// + duration
+		result += sizeof(qint32);
+	}
+	// + thumb loc
+	result += Serialize::imageLocationSize(document->thumbnailLocation());
+	result += sizeof(qint32); // thumbnail_byte_size
+	result += Serialize::imageLocationSize(document->videoThumbnailLocation());
+	result += sizeof(qint32); // video_thumbnail_byte_size
+
+	result += sizeof(qint32) + Serialize::bytearraySize(document->inlineThumbnailBytes());
+
+	return result;
+}
+
+} // namespace Serialize

Telegram/SourceFiles/storage/serialize_document.h

@@ -0,0 +1,48 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#pragma once
+
+#include "data/data_document.h"
+
+namespace Serialize {
+
+class Document {
+public:
+	struct StickerSetInfo {
+		StickerSetInfo(uint64 setId, uint64 accessHash, QString shortName)
+			: setId(setId)
+			, accessHash(accessHash)
+			, shortName(shortName) {
+		}
+		uint64 setId;
+		uint64 accessHash;
+		QString shortName;
+	};
+
+	static void writeToStream(QDataStream &stream, DocumentData *document);
+	static DocumentData *readStickerFromStream(
+		not_null<Main::Session*> session,
+		int streamAppVersion,
+		QDataStream &stream,
+		const StickerSetInfo &info);
+	static DocumentData *readFromStream(
+		not_null<Main::Session*> session,
+		int streamAppVersion,
+		QDataStream &stream);
+	static int sizeInStream(DocumentData *document);
+
+private:
+	static DocumentData *readFromStreamHelper(
+		not_null<Main::Session*> session,
+		int streamAppVersion,
+		QDataStream &stream,
+		const StickerSetInfo *info);
+
+};
+
+} // namespace Serialize

Telegram/SourceFiles/storage/serialize_peer.cpp

@@ -0,0 +1,474 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#include "storage/serialize_peer.h"
+
+#include "storage/serialize_common.h"
+#include "main/main_session.h"
+#include "data/data_channel.h"
+#include "data/data_chat.h"
+#include "data/data_user.h"
+#include "data/data_session.h"
+#include "ui/image/image.h"
+#include "ui/text/format_values.h" // Ui::FormatPhone
+
+namespace Serialize {
+namespace {
+
+constexpr auto kModernImageLocationTag = std::numeric_limits<qint32>::min();
+constexpr auto kVersionTag = uint64(0x77FF'FFFF'FFFF'FFFFULL);
+constexpr auto kVersion = 2;
+
+} // namespace
+
+std::optional<StorageImageLocation> readLegacyStorageImageLocationOrTag(
+		int streamAppVersion,
+		QDataStream &stream) {
+	qint32 width, height, dc, local;
+	quint64 volume, secret;
+	QByteArray fileReference;
+	stream >> width;
+	if (width == kModernImageLocationTag) {
+		return std::nullopt;
+	}
+	stream >> height >> dc >> volume >> local >> secret;
+	if (streamAppVersion >= 1003013) {
+		stream >> fileReference;
+	}
+	if (stream.status() != QDataStream::Ok) {
+		return std::nullopt;
+	}
+	return StorageImageLocation(
+		StorageFileLocation(
+			dc,
+			UserId(0), // self
+			MTP_inputFileLocation(
+				MTP_long(volume),
+				MTP_int(local),
+				MTP_long(secret),
+				MTP_bytes(fileReference))),
+		width,
+		height);
+}
+
+int storageImageLocationSize(const StorageImageLocation &location) {
+	// Modern image location tag + (size + content) of the serialization.
+	return sizeof(qint32) * 2 + location.serializeSize();
+}
+
+void writeStorageImageLocation(
+		QDataStream &stream,
+		const StorageImageLocation &location) {
+	stream << kModernImageLocationTag << location.serialize();
+}
+
+std::optional<StorageImageLocation> readStorageImageLocation(
+		int streamAppVersion,
+		QDataStream &stream) {
+	const auto legacy = readLegacyStorageImageLocationOrTag(
+		streamAppVersion,
+		stream);
+	if (legacy) {
+		return legacy;
+	}
+	auto serialized = QByteArray();
+	stream >> serialized;
+	return (stream.status() == QDataStream::Ok)
+		? StorageImageLocation::FromSerialized(serialized)
+		: std::nullopt;
+}
+
+int imageLocationSize(const ImageLocation &location) {
+	// Modern image location tag + (size + content) of the serialization.
+	return sizeof(qint32) * 2 + location.serializeSize();
+}
+
+void writeImageLocation(QDataStream &stream, const ImageLocation &location) {
+	stream << kModernImageLocationTag << location.serialize();
+}
+
+std::optional<ImageLocation> readImageLocation(
+		int streamAppVersion,
+		QDataStream &stream) {
+	const auto legacy = readLegacyStorageImageLocationOrTag(
+		streamAppVersion,
+		stream);
+	if (legacy) {
+		return ImageLocation(
+			DownloadLocation{ legacy->file() },
+			legacy->width(),
+			legacy->height());
+	}
+	auto serialized = QByteArray();
+	stream >> serialized;
+	return (stream.status() == QDataStream::Ok)
+		? ImageLocation::FromSerialized(serialized)
+		: std::nullopt;
+}
+
+uint32 peerSize(not_null<PeerData*> peer) {
+	uint32 result = sizeof(quint64) // id
+		+ sizeof(quint64) // version tag
+		+ sizeof(qint32) // version
+		+ sizeof(quint64) // userpic photo id
+		+ imageLocationSize(peer->userpicLocation())
+		+ sizeof(qint32); // userpic has video
+	if (const auto user = peer->asUser()) {
+		const auto botInlinePlaceholder = user->isBot()
+			? user->botInfo->inlinePlaceholder
+			: QString();
+		result += stringSize(user->firstName)
+			+ stringSize(user->lastName)
+			+ stringSize(user->phone())
+			+ stringSize(user->username())
+			+ sizeof(quint64) // access
+			+ sizeof(qint32) // flags
+			+ stringSize(botInlinePlaceholder)
+			+ sizeof(quint32) // lastseen
+			+ sizeof(qint32) // contact
+			+ sizeof(qint32); // botInfoVersion
+	} else if (const auto chat = peer->asChat()) {
+		result += stringSize(chat->name())
+			+ sizeof(qint32) // count
+			+ sizeof(qint32) // date
+			+ sizeof(qint32) // version
+			+ sizeof(qint32) // creator id 1
+			+ sizeof(qint32) // creator id 2
+			+ sizeof(quint32) // flags
+			+ stringSize(chat->inviteLink());
+	} else if (const auto channel = peer->asChannel()) {
+		result += stringSize(channel->name())
+			+ sizeof(quint64) // access
+			+ sizeof(qint32) // date
+			+ sizeof(qint32) // version
+			+ sizeof(qint32) // old forbidden
+			+ sizeof(quint32) // flags
+			+ stringSize(channel->inviteLink());
+	}
+	return result;
+}
+
+void writePeer(QDataStream &stream, not_null<PeerData*> peer) {
+	stream
+		<< SerializePeerId(peer->id)
+		<< quint64(kVersionTag)
+		<< qint32(kVersion)
+		<< quint64(peer->userpicPhotoId());
+	writeImageLocation(stream, peer->userpicLocation());
+	stream << qint32(peer->userpicHasVideo() ? 1 : 0);
+	if (const auto user = peer->asUser()) {
+		const auto botInlinePlaceholder = user->isBot()
+			? user->botInfo->inlinePlaceholder
+			: QString();
+		stream
+			<< user->firstName
+			<< user->lastName
+			<< user->phone()
+			<< user->username()
+			<< quint64(user->accessHash())
+			<< qint32(user->flags())
+			<< botInlinePlaceholder
+			<< quint32(user->lastseen().serialize())
+			<< qint32(user->isContact() ? 1 : 0)
+			<< qint32(user->isBot() ? user->botInfo->version : -1);
+	} else if (const auto chat = peer->asChat()) {
+		auto field1 = qint32(uint32(chat->creator.bare & 0xFFFFFFFFULL));
+		auto field2 = qint32(uint32(chat->creator.bare >> 32) << 8);
+		stream
+			<< chat->name()
+			<< qint32(chat->count)
+			<< qint32(chat->date)
+			<< qint32(chat->version())
+			<< field1
+			<< field2
+			<< quint32(chat->flags())
+			<< chat->inviteLink();
+	} else if (const auto channel = peer->asChannel()) {
+		stream
+			<< channel->name()
+			<< quint64(channel->accessHash())
+			<< qint32(channel->date)
+			<< qint32(0) // legacy - version
+			<< qint32(0)
+			<< quint32(channel->flags())
+			<< channel->inviteLink();
+	}
+}
+
+PeerData *readPeer(
+		not_null<Main::Session*> session,
+		int streamAppVersion,
+		QDataStream &stream) {
+	quint64 peerIdSerialized = 0, versionTag = 0, photoId = 0;
+	qint32 version = 0, photoHasVideo = 0;
+	stream >> peerIdSerialized >> versionTag;
+	const auto peerId = DeserializePeerId(peerIdSerialized);
+	if (!peerId) {
+		return nullptr;
+	}
+	if (versionTag == kVersionTag) {
+		stream >> version >> photoId;
+	} else {
+		photoId = versionTag;
+	}
+
+	const auto userpic = readImageLocation(streamAppVersion, stream);
+	auto userpicAccessHash = uint64(0);
+	if (!userpic) {
+		return nullptr;
+	}
+	if (version > 0) {
+		stream >> photoHasVideo;
+	}
+
+	const auto selfId = session->userPeerId();
+	const auto loaded = (peerId == selfId)
+		? session->user().get()
+		: session->data().peerLoaded(peerId);
+	const auto apply = !loaded || !loaded->isLoaded();
+	const auto result = loaded ? loaded : session->data().peer(peerId).get();
+	if (apply) {
+		result->setLoadedStatus(PeerData::LoadedStatus::Normal);
+	}
+	if (const auto user = result->asUser()) {
+		QString first, last, phone, username, inlinePlaceholder;
+		quint64 access;
+		qint32 flags = 0, contact, botInfoVersion;
+		quint32 lastseen;
+		stream >> first >> last >> phone >> username >> access;
+		if (streamAppVersion >= 9012) {
+			stream >> flags;
+		}
+		if (streamAppVersion >= 9016) {
+			stream >> inlinePlaceholder;
+		}
+		stream >> lastseen >> contact >> botInfoVersion;
+
+		userpicAccessHash = access;
+
+		if (apply) {
+			const auto showPhone = !user->isServiceUser()
+				&& (user->id != selfId)
+				&& (contact <= 0);
+			const auto pname = (showPhone && !phone.isEmpty())
+				? Ui::FormatPhone(phone)
+				: QString();
+
+			user->setPhone(phone);
+			user->setName(first, last, pname, username);
+			if (streamAppVersion >= 2008007) {
+				user->setFlags(UserDataFlags::from_raw(flags));
+			} else {
+				using Saved = MTPDuser::Flag;
+				using Flag = UserDataFlag;
+				struct Conversion {
+					Saved saved;
+					Flag flag;
+				};
+				const auto conversions = {
+					Conversion{ Saved::f_deleted, Flag::Deleted },
+					Conversion{ Saved::f_verified, Flag::Verified },
+					Conversion{ Saved::f_scam, Flag::Scam },
+					Conversion{ Saved::f_fake, Flag::Fake },
+					Conversion{ Saved::f_bot_inline_geo, Flag::BotInlineGeo },
+					Conversion{ Saved::f_support, Flag::Support },
+					Conversion{ Saved::f_contact, Flag::Contact },
+					Conversion{ Saved::f_mutual_contact, Flag::MutualContact },
+				};
+				auto flagsMask = Flag() | Flag();
+				auto flagsSet = Flag() | Flag();
+				for (const auto &conversion : conversions) {
+					flagsMask |= conversion.flag;
+					if (flags & int(conversion.saved)) {
+						flagsSet |= conversion.flag;
+					}
+				}
+				user->setFlags((user->flags() & ~flagsMask) | flagsSet);
+			}
+			user->setAccessHash(access);
+			user->updateLastseen((version > 1)
+				? Data::LastseenStatus::FromSerialized(lastseen)
+				: Data::LastseenStatus::FromLegacy(lastseen));
+			user->setIsContact(contact == 1);
+			user->setBotInfoVersion(botInfoVersion);
+			if (!inlinePlaceholder.isEmpty() && user->isBot()) {
+				user->botInfo->inlinePlaceholder = inlinePlaceholder;
+			}
+		}
+	} else if (const auto chat = result->asChat()) {
+		QString name, inviteLink;
+		qint32 count, date, version, field1, field2;
+		quint32 flags;
+		stream
+			>> name
+			>> count
+			>> date
+			>> version
+			>> field1
+			>> field2
+			>> flags
+			>> inviteLink;
+		if (apply) {
+			const auto creator = UserId(
+				BareId(uint32(field1)) | (BareId(uint32(field2) >> 8) << 32));
+			chat->setName(name);
+			chat->count = count;
+			chat->date = date;
+
+			// We don't save participants, admin status and banned rights.
+			// So we don't restore the version field, info is still unknown.
+			chat->setVersion(0);
+
+			if (streamAppVersion >= 2008007) {
+				chat->setFlags(ChatDataFlags::from_raw(flags));
+			} else {
+				const auto oldForbidden = ((uint32(field2) & 0xFF) == 1);
+
+				using Saved = MTPDchat::Flag;
+				using Flag = ChatDataFlag;
+				struct Conversion {
+					Saved saved;
+					Flag flag;
+				};
+				const auto conversions = {
+					Conversion{ Saved::f_left, Flag::Left },
+					Conversion{ Saved::f_creator, Flag::Creator },
+					Conversion{ Saved::f_deactivated, Flag::Deactivated },
+					Conversion{ Saved(1U << 31), Flag::Forbidden },
+					Conversion{ Saved::f_call_active, Flag::CallActive },
+					Conversion{ Saved::f_call_not_empty, Flag::CallNotEmpty },
+				};
+				auto flagsMask = Flag() | Flag();
+				auto flagsSet = Flag() | Flag();
+				if (streamAppVersion >= 9012) {
+					for (const auto &conversion : conversions) {
+						flagsMask |= conversion.flag;
+						if (flags & int(conversion.saved)) {
+							flagsSet |= conversion.flag;
+						}
+					}
+				} else {
+					// flags was haveLeft
+					if (flags == 1) {
+						flagsSet |= Flag::Left;
+					}
+				}
+				if (oldForbidden) {
+					flagsSet |= Flag::Forbidden;
+				}
+				chat->setFlags((chat->flags() & ~flagsMask) | flagsSet);
+			}
+
+			chat->creator = creator;
+			chat->setInviteLink(inviteLink);
+		}
+	} else if (const auto channel = result->asChannel()) {
+		QString name, inviteLink;
+		quint64 access;
+		qint32 date, version, oldForbidden;
+		quint32 flags;
+		stream
+			>> name
+			>> access
+			>> date
+			>> version
+			>> oldForbidden
+			>> flags
+			>> inviteLink;
+
+		userpicAccessHash = access;
+
+		if (apply) {
+			channel->setName(name, QString());
+			channel->setAccessHash(access);
+			channel->date = date;
+
+			if (streamAppVersion >= 2008007) {
+				channel->setFlags(ChannelDataFlags::from_raw(flags));
+			} else {
+				using Saved = MTPDchannel::Flag;
+				using Flag = ChannelDataFlag;
+				struct Conversion {
+					Saved saved;
+					Flag flag;
+				};
+				const auto conversions = {
+					Conversion{ Saved::f_broadcast, Flag::Broadcast },
+					Conversion{ Saved::f_verified, Flag::Verified},
+					Conversion{ Saved::f_scam, Flag::Scam},
+					Conversion{ Saved::f_fake, Flag::Fake},
+					Conversion{ Saved::f_megagroup, Flag::Megagroup},
+					Conversion{ Saved::f_gigagroup, Flag::Gigagroup},
+					Conversion{ Saved::f_username, Flag::Username},
+					Conversion{ Saved::f_signatures, Flag::Signatures},
+					Conversion{ Saved::f_has_link, Flag::HasLink},
+					Conversion{
+						Saved::f_slowmode_enabled,
+						Flag::SlowmodeEnabled },
+					Conversion{ Saved::f_call_active, Flag::CallActive },
+					Conversion{ Saved::f_call_not_empty, Flag::CallNotEmpty },
+					Conversion{ Saved(1U << 31), Flag::Forbidden },
+					Conversion{ Saved::f_left, Flag::Left },
+					Conversion{ Saved::f_creator, Flag::Creator },
+				};
+				auto flagsMask = Flag() | Flag();
+				auto flagsSet = Flag() | Flag();
+				for (const auto &conversion : conversions) {
+					flagsMask |= conversion.flag;
+					if (flags & int(conversion.saved)) {
+						flagsSet |= conversion.flag;
+					}
+				}
+				if (oldForbidden) {
+					flagsSet |= Flag::Forbidden;
+				}
+				channel->setFlags((channel->flags() & ~flagsMask) | flagsSet);
+			}
+
+			channel->setInviteLink(inviteLink);
+		}
+	}
+	if (apply) {
+		const auto location = userpic->convertToModernPeerPhoto(
+			result->id.value,
+			userpicAccessHash,
+			photoId);
+		result->setUserpic(photoId, location, (photoHasVideo == 1));
+	}
+	return result;
+}
+
+QString peekUserPhone(int streamAppVersion, QDataStream &stream) {
+	quint64 peerIdSerialized = 0, versionTag = 0, photoId = 0;
+	qint32 version = 0, photoHasVideo = 0;
+	stream >> peerIdSerialized >> versionTag;
+	const auto peerId = DeserializePeerId(peerIdSerialized);
+	DEBUG_LOG(("peekUserPhone.id: %1").arg(peerId.value));
+	if (!peerId || !peerIsUser(peerId)) {
+		return nullptr;
+	}
+	if (versionTag == kVersionTag) {
+		stream >> version >> photoId;
+	} else {
+		photoId = versionTag;
+	}
+
+	if (!readImageLocation(streamAppVersion, stream)) {
+		return nullptr;
+	}
+	if (version > 0) {
+		stream >> photoHasVideo;
+	}
+
+	QString first, last, phone;
+	stream >> first >> last >> phone;
+	DEBUG_LOG(("peekUserPhone.data: %1 %2 %3"
+		).arg(first, last, phone));
+	return phone;
+}
+
+} // namespace Serialize

Telegram/SourceFiles/storage/serialize_peer.h

@@ -0,0 +1,44 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#pragma once
+
+namespace Main {
+class Session;
+} // namespace Main
+
+namespace Serialize {
+
+int storageImageLocationSize(const StorageImageLocation &location);
+void writeStorageImageLocation(
+	QDataStream &stream,
+	const StorageImageLocation &location);
+
+// NB! This method can return StorageFileLocation with Type::Generic!
+// The reader should discard it or convert to one of the valid modern types.
+std::optional<StorageImageLocation> readStorageImageLocation(
+	int streamAppVersion,
+	QDataStream &stream);
+
+int imageLocationSize(const ImageLocation &location);
+void writeImageLocation(QDataStream &stream, const ImageLocation &location);
+
+// NB! This method can return StorageFileLocation with Type::Generic!
+// The reader should discard it or convert to one of the valid modern types.
+std::optional<ImageLocation> readImageLocation(
+	int streamAppVersion,
+	QDataStream &stream);
+
+uint32 peerSize(not_null<PeerData*> peer);
+void writePeer(QDataStream &stream, not_null<PeerData*> peer);
+PeerData *readPeer(
+	not_null<Main::Session*> session,
+	int streamAppVersion,
+	QDataStream &stream);
+QString peekUserPhone(int streamAppVersion, QDataStream &stream);
+
+} // namespace Serialize

Telegram/SourceFiles/storage/storage_account.h

@@ -0,0 +1,400 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#pragma once
+
+#include "base/timer.h"
+#include "base/flags.h"
+#include "storage/cache/storage_cache_database.h"
+#include "data/stickers/data_stickers_set.h"
+#include "data/data_drafts.h"
+#include "webview/webview_common.h"
+
+class History;
+
+namespace Core {
+class FileLocation;
+} // namespace Core
+
+namespace Export {
+struct Settings;
+} // namespace Export
+
+namespace Main {
+class Account;
+class SessionSettings;
+} // namespace Main
+
+namespace Data {
+class WallPaper;
+} // namespace Data
+
+namespace MTP {
+class Config;
+class AuthKey;
+using AuthKeyPtr = std::shared_ptr<AuthKey>;
+} // namespace MTP
+
+namespace Storage {
+namespace details {
+struct ReadSettingsContext;
+struct FileReadDescriptor;
+} // namespace details
+
+class EncryptionKey;
+
+using FileKey = quint64;
+
+enum class StartResult : uchar;
+
+struct MessageDraft {
+	FullReplyTo reply;
+	SuggestOptions suggest;
+	TextWithTags textWithTags;
+	Data::WebPageDraft webpage;
+};
+
+struct MessageDraftSource {
+	Fn<MessageDraft()> draft;
+	Fn<MessageCursor()> cursor;
+};
+
+class Account final {
+public:
+	Account(not_null<Main::Account*> owner, const QString &dataName);
+	~Account();
+
+	[[nodiscard]] StartResult legacyStart(const QByteArray &passcode);
+	[[nodiscard]] std::unique_ptr<MTP::Config> start(
+		MTP::AuthKeyPtr localKey);
+	void startAdded(MTP::AuthKeyPtr localKey);
+	[[nodiscard]] int oldMapVersion() const {
+		return _oldMapVersion;
+	}
+
+	[[nodiscard]] QString tempDirectory() const;
+	[[nodiscard]] QString supportModePath() const;
+
+	[[nodiscard]] MTP::AuthKeyPtr peekLegacyLocalKey() const {
+		return _localKey;
+	}
+
+	void writeSessionSettings();
+	void writeMtpData();
+	void writeMtpConfig();
+
+	void registerDraftSource(
+		not_null<History*> history,
+		Data::DraftKey key,
+		MessageDraftSource source);
+	void unregisterDraftSource(
+		not_null<History*> history,
+		Data::DraftKey key);
+	void writeDrafts(not_null<History*> history);
+	void readDraftsWithCursors(not_null<History*> history);
+	void writeDraftCursors(not_null<History*> history);
+	[[nodiscard]] bool hasDraftCursors(PeerId peerId);
+	[[nodiscard]] bool hasDraft(PeerId peerId);
+
+	void writeFileLocation(
+		MediaKey location,
+		const Core::FileLocation &local);
+	[[nodiscard]] Core::FileLocation readFileLocation(MediaKey location);
+	void removeFileLocation(MediaKey location);
+
+	void updateDownloads(Fn<std::optional<QByteArray>()> downloadsSerialize);
+	[[nodiscard]] QByteArray downloadsSerialized() const;
+
+	[[nodiscard]] EncryptionKey cacheKey() const;
+	[[nodiscard]] QString cachePath() const;
+	[[nodiscard]] Cache::Database::Settings cacheSettings() const;
+	void updateCacheSettings(
+		Cache::Database::SettingsUpdate &update,
+		Cache::Database::SettingsUpdate &updateBig);
+
+	[[nodiscard]] EncryptionKey cacheBigFileKey() const;
+	[[nodiscard]] QString cacheBigFilePath() const;
+	[[nodiscard]] Cache::Database::Settings cacheBigFileSettings() const;
+
+	void writeInstalledStickers();
+	void writeFeaturedStickers();
+	void writeRecentStickers();
+	void writeFavedStickers();
+	void writeArchivedStickers();
+	void writeArchivedMasks();
+	void readInstalledStickers();
+	void readFeaturedStickers();
+	void readRecentStickers();
+	void readFavedStickers();
+	void readArchivedStickers();
+	void readArchivedMasks();
+	void writeSavedGifs();
+	void readSavedGifs();
+	void writeInstalledMasks();
+	void writeRecentMasks();
+	void readInstalledMasks();
+	void readRecentMasks();
+	void writeInstalledCustomEmoji();
+	void writeFeaturedCustomEmoji();
+	void readInstalledCustomEmoji();
+	void readFeaturedCustomEmoji();
+
+	void writeRecentHashtagsAndBots();
+	void readRecentHashtagsAndBots();
+	void saveRecentSentHashtags(const QString &text);
+	void saveRecentSearchHashtags(const QString &text);
+
+	void writeExportSettings(const Export::Settings &settings);
+	[[nodiscard]] Export::Settings readExportSettings();
+
+	void setMediaLastPlaybackPosition(DocumentId id, crl::time time);
+	[[nodiscard]] crl::time mediaLastPlaybackPosition(DocumentId id) const;
+
+	void writeSearchSuggestionsDelayed();
+	void writeSearchSuggestionsIfNeeded();
+	void writeSearchSuggestions();
+	void readSearchSuggestions();
+
+	void writeSelf();
+
+	// Read self is special, it can't get session from account, because
+	// it is not really there yet - it is still being constructed.
+	void readSelf(
+		not_null<Main::Session*> session,
+		const QByteArray& serialized,
+		int32 streamVersion);
+
+	void markPeerTrustedOpenGame(PeerId peerId);
+	[[nodiscard]] bool isPeerTrustedOpenGame(PeerId peerId);
+	void markPeerTrustedPayment(PeerId peerId);
+	[[nodiscard]] bool isPeerTrustedPayment(PeerId peerId);
+	void markPeerTrustedOpenWebView(PeerId peerId);
+	[[nodiscard]] bool isPeerTrustedOpenWebView(PeerId peerId);
+	void markPeerTrustedPayForMessage(PeerId peerId, int starsPerMessage);
+	[[nodiscard]] bool isPeerTrustedPayForMessage(
+		PeerId peerId,
+		int starsPerMessage);
+	[[nodiscard]] bool peerTrustedPayForMessageRead() const;
+	[[nodiscard]] bool hasPeerTrustedPayForMessageEntry(PeerId peerId) const;
+	void clearPeerTrustedPayForMessage(PeerId peerId);
+
+	template <typename Type, typename Other>
+	void writePref(std::string_view key, Other &&value) {
+		writePrefImpl<Type>(key, std::forward<Other>(value));
+	}
+	void clearPref(std::string_view key);
+
+	template <typename Type, typename Other = Type>
+	[[nodiscard]] Type readPref(
+			std::string_view key,
+			Other &&fallback = Type()) {
+		return readPrefImpl<Type>(key).value_or(std::forward<Other>(fallback));
+	}
+
+	void enforceModernStorageIdBots();
+	[[nodiscard]] Webview::StorageId resolveStorageIdBots();
+	[[nodiscard]] Webview::StorageId resolveStorageIdOther();
+
+	[[nodiscard]] QImage readRoundPlaceholder();
+	void writeRoundPlaceholder(const QImage &placeholder);
+
+	[[nodiscard]] QByteArray readInlineBotsDownloads();
+	void writeInlineBotsDownloads(const QByteArray &bytes);
+
+	void writeBotStorage(PeerId botId, const QByteArray &serialized);
+	[[nodiscard]] QByteArray readBotStorage(PeerId botId);
+
+	[[nodiscard]] bool encrypt(
+		const void *src,
+		void *dst,
+		uint32 len,
+		const void *key128) const;
+	[[nodiscard]] bool decrypt(
+		const void *src,
+		void *dst,
+		uint32 len,
+		const void *key128) const;
+
+	void reset();
+
+private:
+	enum class ReadMapResult {
+		Success,
+		IncorrectPasscode,
+		Failed,
+	};
+	enum class PeerTrustFlag : uchar {
+		NoOpenGame        = (1 << 0),
+		Payment           = (1 << 1),
+		OpenWebView       = (1 << 2),
+	};
+	friend inline constexpr bool is_flag_type(PeerTrustFlag) { return true; };
+
+	[[nodiscard]] base::flat_set<QString> collectGoodNames() const;
+	[[nodiscard]] auto prepareReadSettingsContext() const
+		-> details::ReadSettingsContext;
+
+	ReadMapResult readMapWith(
+		MTP::AuthKeyPtr localKey,
+		const QByteArray &legacyPasscode = QByteArray());
+	void clearLegacyFiles();
+	void writeMapDelayed();
+	void writeMapQueued();
+	void writeMap();
+
+	void readLocations();
+	void writeLocations();
+	void writeLocationsQueued();
+	void writeLocationsDelayed();
+
+	void readPrefs();
+	void writePrefs();
+	void writePrefsDelayed();
+
+	std::unique_ptr<Main::SessionSettings> readSessionSettings();
+	void writeSessionSettings(Main::SessionSettings *stored);
+
+	std::unique_ptr<MTP::Config> readMtpConfig();
+	void readMtpData();
+	std::unique_ptr<Main::SessionSettings> applyReadContext(
+		details::ReadSettingsContext &&context);
+
+	void readDraftCursors(PeerId peerId, Data::HistoryDrafts &map);
+	void readDraftCursorsLegacy(
+		PeerId peerId,
+		details::FileReadDescriptor &draft,
+		quint64 draftPeerSerialized,
+		Data::HistoryDrafts &map);
+	void clearDraftCursors(PeerId peerId);
+	void readDraftsWithCursorsLegacy(
+		not_null<History*> history,
+		details::FileReadDescriptor &draft,
+		quint64 draftPeerSerialized);
+
+	void writeStickerSet(
+		QDataStream &stream,
+		const Data::StickersSet &set);
+	template <typename CheckSet>
+	void writeStickerSets(
+		FileKey &stickersKey,
+		CheckSet checkSet,
+		const Data::StickersSetsOrder &order);
+	void readStickerSets(
+		FileKey &stickersKey,
+		Data::StickersSetsOrder *outOrder = nullptr,
+		Data::StickersSetFlags readingFlags = 0);
+	void importOldRecentStickers();
+
+	void readTrustedPeers();
+	void writeTrustedPeers();
+
+	void readMediaLastPlaybackPositions();
+	void writeMediaLastPlaybackPositions();
+
+	std::optional<RecentHashtagPack> saveRecentHashtags(
+		Fn<RecentHashtagPack()> getPack,
+		const QString &text);
+
+	template <typename Type>
+	void writePrefImpl(std::string_view key, Type value);
+
+	template <typename Type>
+	[[nodiscard]] std::optional<Type> readPrefImpl(std::string_view key);
+
+	void writePrefGeneric(std::string_view key, const QByteArray &value);
+	[[nodiscard]] std::optional<QByteArray> readPrefGeneric(
+		std::string_view key);
+
+	const not_null<Main::Account*> _owner;
+	const QString _dataName;
+	const FileKey _dataNameKey = 0;
+	const QString _basePath;
+	const QString _tempPath;
+	const QString _databasePath;
+
+	MTP::AuthKeyPtr _localKey;
+
+	base::flat_map<PeerId, FileKey> _draftsMap;
+	base::flat_map<PeerId, FileKey> _draftCursorsMap;
+	base::flat_map<PeerId, bool> _draftsNotReadMap;
+	base::flat_map<
+		not_null<History*>,
+		base::flat_map<Data::DraftKey, MessageDraftSource>> _draftSources;
+	base::flat_map<PeerId, FileKey> _botStoragesMap;
+	base::flat_map<PeerId, bool> _botStoragesNotReadMap;
+
+	QMultiMap<MediaKey, Core::FileLocation> _fileLocations;
+	QMap<QString, QPair<MediaKey, Core::FileLocation>> _fileLocationPairs;
+	QMap<MediaKey, MediaKey> _fileLocationAliases;
+
+	QByteArray _downloadsSerialized;
+	Fn<std::optional<QByteArray>()> _downloadsSerialize;
+
+	FileKey _prefsKey = 0;
+	FileKey _locationsKey = 0;
+	FileKey _trustedPeersKey = 0;
+	FileKey _installedStickersKey = 0;
+	FileKey _featuredStickersKey = 0;
+	FileKey _recentStickersKey = 0;
+	FileKey _favedStickersKey = 0;
+	FileKey _archivedStickersKey = 0;
+	FileKey _archivedMasksKey = 0;
+	FileKey _savedGifsKey = 0;
+	FileKey _recentStickersKeyOld = 0;
+	FileKey _legacyBackgroundKeyDay = 0;
+	FileKey _legacyBackgroundKeyNight = 0;
+	FileKey _settingsKey = 0;
+	FileKey _recentHashtagsAndBotsKey = 0;
+	FileKey _exportSettingsKey = 0;
+	FileKey _installedMasksKey = 0;
+	FileKey _recentMasksKey = 0;
+	FileKey _installedCustomEmojiKey = 0;
+	FileKey _featuredCustomEmojiKey = 0;
+	FileKey _archivedCustomEmojiKey = 0;
+	FileKey _searchSuggestionsKey = 0;
+	FileKey _roundPlaceholderKey = 0;
+	FileKey _inlineBotsDownloadsKey = 0;
+	FileKey _mediaLastPlaybackPositionsKey = 0;
+
+	qint64 _cacheTotalSizeLimit = 0;
+	qint64 _cacheBigFileTotalSizeLimit = 0;
+	qint32 _cacheTotalTimeLimit = 0;
+	qint32 _cacheBigFileTotalTimeLimit = 0;
+
+	base::flat_map<PeerId, base::flags<PeerTrustFlag>> _trustedPeers;
+	base::flat_map<PeerId, int> _trustedPayPerMessage;
+	bool _trustedPeersRead = false;
+	bool _readingUserSettings = false;
+	bool _recentHashtagsAndBotsWereRead = false;
+	bool _searchSuggestionsRead = false;
+	bool _inlineBotsDownloadsRead = false;
+	bool _mediaLastPlaybackPositionsRead = false;
+
+	std::vector<std::pair<DocumentId, crl::time>> _mediaLastPlaybackPosition;
+
+	Webview::StorageId _webviewStorageIdBots;
+	Webview::StorageId _webviewStorageIdOther;
+
+	base::flat_map<QByteArray, QByteArray> _prefs;
+
+	int _oldMapVersion = 0;
+
+	base::Timer _writeMapTimer;
+	base::Timer _writePrefsTimer;
+	base::Timer _writeLocationsTimer;
+	base::Timer _writeSearchSuggestionsTimer;
+	bool _mapChanged = false;
+	bool _prefsChanged = false;
+	bool _locationsChanged = false;
+
+	QImage _roundPlaceholder;
+
+};
+
+[[nodiscard]] Webview::StorageId TonSiteStorageId();
+
+} // namespace Storage

Telegram/SourceFiles/storage/storage_cloud_blob.cpp

@@ -0,0 +1,151 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#include "storage/storage_cloud_blob.h"
+
+#include "base/zlib_help.h"
+#include "lang/lang_keys.h"
+#include "ui/text/format_values.h"
+#include "main/main_account.h"
+#include "main/main_session.h"
+
+namespace Storage::CloudBlob {
+
+namespace {
+
+QByteArray ReadFinalFile(const QString &path) {
+	constexpr auto kMaxZipSize = 10 * 1024 * 1024;
+	auto file = QFile(path);
+	if (file.size() > kMaxZipSize || !file.open(QIODevice::ReadOnly)) {
+		return QByteArray();
+	}
+	return file.readAll();
+}
+
+bool ExtractZipFile(zlib::FileToRead &zip, const QString path) {
+	constexpr auto kMaxSize = 25 * 1024 * 1024;
+	const auto content = zip.readCurrentFileContent(kMaxSize);
+	if (content.isEmpty() || zip.error() != UNZ_OK) {
+		return false;
+	}
+	auto file = QFile(path);
+	return file.open(QIODevice::WriteOnly)
+		&& (file.write(content) == content.size());
+}
+
+} // namespace
+
+bool UnpackBlob(
+		const QString &path,
+		const QString &folder,
+		Fn<bool(const QString &)> checkNameCallback) {
+	const auto bytes = ReadFinalFile(path);
+	if (bytes.isEmpty()) {
+		return false;
+	}
+	auto zip = zlib::FileToRead(bytes);
+	if (zip.goToFirstFile() != UNZ_OK) {
+		return false;
+	}
+	do {
+		const auto name = zip.getCurrentFileName();
+		const auto path = folder + '/' + name;
+		if (checkNameCallback(name) && !ExtractZipFile(zip, path)) {
+			return false;
+		}
+
+		const auto jump = zip.goToNextFile();
+		if (jump == UNZ_END_OF_LIST_OF_FILE) {
+			break;
+		} else if (jump != UNZ_OK) {
+			return false;
+		}
+	} while (true);
+	return true;
+}
+
+QString StateDescription(const BlobState &state, tr::phrase<> activeText) {
+	return v::match(state, [](const Available &data) {
+		return tr::lng_emoji_set_download(
+			tr::now,
+			lt_size,
+			Ui::FormatSizeText(data.size));
+	}, [](const Ready &data) -> QString {
+		return tr::lng_emoji_set_ready(tr::now);
+	}, [&](const Active &data) -> QString {
+		return activeText(tr::now);
+	}, [](const Loading &data) {
+		const auto percent = (data.size > 0)
+			? std::clamp((data.already * 100) / float64(data.size), 0., 100.)
+			: 0.;
+		return tr::lng_emoji_set_loading(
+			tr::now,
+			lt_percent,
+			QString::number(int(base::SafeRound(percent))) + '%',
+			lt_progress,
+			Ui::FormatDownloadText(data.already, data.size));
+	}, [](const Failed &data) {
+		return tr::lng_attach_failed(tr::now);
+	});
+}
+
+BlobLoader::BlobLoader(
+	QObject *parent,
+	not_null<Main::Session*> session,
+	int id,
+	MTP::DedicatedLoader::Location location,
+	const QString &folder,
+	int64 size)
+: QObject(parent)
+, _folder(folder)
+, _id(id)
+, _state(Loading{ 0, size })
+, _mtproto(session.get()) {
+	const auto ready = [=](std::unique_ptr<MTP::DedicatedLoader> loader) {
+		if (loader) {
+			setImplementation(std::move(loader));
+		} else {
+			fail();
+		}
+	};
+	MTP::StartDedicatedLoader(&_mtproto, location, _folder, ready);
+}
+
+int BlobLoader::id() const {
+	return _id;
+}
+
+rpl::producer<BlobState> BlobLoader::state() const {
+	return _state.value();
+}
+
+void BlobLoader::setImplementation(
+		std::unique_ptr<MTP::DedicatedLoader> loader) {
+	_implementation = std::move(loader);
+	_state = _implementation->progress(
+	) | rpl::map([](const Loading &state) {
+		return BlobState(state);
+	});
+	_implementation->failed(
+	) | rpl::on_next([=] {
+		fail();
+	}, _implementation->lifetime());
+
+	_implementation->ready(
+	) | rpl::on_next([=](const QString &filepath) {
+		unpack(filepath);
+	}, _implementation->lifetime());
+
+	QDir(_folder).removeRecursively();
+	_implementation->start();
+}
+
+void BlobLoader::fail() {
+	_state = Failed();
+}
+
+} // namespace Storage::CloudBlob

Telegram/SourceFiles/storage/storage_cloud_blob.h

@@ -0,0 +1,115 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#pragma once
+
+#include "mtproto/dedicated_file_loader.h"
+
+namespace tr {
+template <typename ...>
+struct phrase;
+} // namespace tr
+
+namespace Main {
+class Session;
+} // namespace Main
+
+namespace Storage::CloudBlob {
+
+constexpr auto kCloudLocationUsername = "tdhbcfiles"_cs;
+
+struct Blob {
+	int id = 0;
+	int postId = 0;
+	int64 size = 0;
+	QString name;
+};
+
+struct Available {
+	int64 size = 0;
+
+	inline bool operator<(const Available &other) const {
+		return size < other.size;
+	}
+	inline bool operator==(const Available &other) const {
+		return size == other.size;
+	}
+};
+struct Ready {
+	inline bool operator<(const Ready &other) const {
+		return false;
+	}
+	inline bool operator==(const Ready &other) const {
+		return true;
+	}
+};
+struct Active {
+	inline bool operator<(const Active &other) const {
+		return false;
+	}
+	inline bool operator==(const Active &other) const {
+		return true;
+	}
+};
+struct Failed {
+	inline bool operator<(const Failed &other) const {
+		return false;
+	}
+	inline bool operator==(const Failed &other) const {
+		return true;
+	}
+};
+
+using Loading = MTP::DedicatedLoader::Progress;
+using BlobState = std::variant<
+	Available,
+	Ready,
+	Active,
+	Failed,
+	Loading>;
+
+bool UnpackBlob(
+	const QString &path,
+	const QString &folder,
+	Fn<bool(const QString &)> checkNameCallback);
+
+QString StateDescription(const BlobState &state, tr::phrase<> activeText);
+
+class BlobLoader : public QObject {
+public:
+	BlobLoader(
+		QObject *parent,
+		not_null<Main::Session*> session,
+		int id,
+		MTP::DedicatedLoader::Location location,
+		const QString &folder,
+		int64 size);
+
+	int id() const;
+
+	rpl::producer<BlobState> state() const;
+	virtual void destroy() = 0;
+	virtual void unpack(const QString &path) = 0;
+
+protected:
+	virtual void fail();
+
+	const QString _folder;
+
+private:
+	void setImplementation(std::unique_ptr<MTP::DedicatedLoader> loader);
+
+	int _id = 0;
+	rpl::variable<BlobState> _state;
+
+	MTP::WeakInstance _mtproto;
+
+	std::unique_ptr<MTP::DedicatedLoader> _implementation;
+
+};
+
+} // namespace Storage::CloudBlob

Telegram/SourceFiles/storage/storage_domain.cpp

@@ -0,0 +1,279 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#include "storage/storage_domain.h"
+
+#include "storage/details/storage_file_utilities.h"
+#include "storage/serialize_common.h"
+#include "mtproto/mtproto_config.h"
+#include "main/main_domain.h"
+#include "main/main_account.h"
+#include "base/random.h"
+
+namespace Storage {
+namespace {
+
+using namespace details;
+
+[[nodiscard]] QString BaseGlobalPath() {
+	return cWorkingDir() + u"tdata/"_q;
+}
+
+[[nodiscard]] QString ComputeKeyName(const QString &dataName) {
+	// We dropped old test authorizations when migrated to multi auth.
+	//return "key_" + dataName + (cTestMode() ? "[test]" : "");
+	return "key_" + dataName;
+}
+
+} // namespace
+
+Domain::Domain(not_null<Main::Domain*> owner, const QString &dataName)
+: _owner(owner)
+, _dataName(dataName) {
+}
+
+Domain::~Domain() = default;
+
+StartResult Domain::start(const QByteArray &passcode) {
+	const auto modern = startModern(passcode);
+	if (modern == StartModernResult::Success) {
+		if (_oldVersion < AppVersion) {
+			writeAccounts();
+		}
+		return StartResult::Success;
+	} else if (modern == StartModernResult::IncorrectPasscode) {
+		return StartResult::IncorrectPasscode;
+	} else if (modern == StartModernResult::Failed) {
+		startFromScratch();
+		return StartResult::Success;
+	}
+	auto legacy = std::make_unique<Main::Account>(_owner, _dataName, 0);
+	const auto result = legacy->legacyStart(passcode);
+	if (result == StartResult::Success) {
+		_oldVersion = legacy->local().oldMapVersion();
+		startWithSingleAccount(passcode, std::move(legacy));
+	}
+	return result;
+}
+
+void Domain::startAdded(
+		not_null<Main::Account*> account,
+		std::unique_ptr<MTP::Config> config) {
+	Expects(_localKey != nullptr);
+
+	account->prepareToStartAdded(_localKey);
+	account->start(std::move(config));
+}
+
+void Domain::startWithSingleAccount(
+		const QByteArray &passcode,
+		std::unique_ptr<Main::Account> account) {
+	Expects(account != nullptr);
+
+	if (auto localKey = account->local().peekLegacyLocalKey()) {
+		_localKey = std::move(localKey);
+		encryptLocalKey(passcode);
+		account->start(nullptr);
+	} else {
+		generateLocalKey();
+		account->start(account->prepareToStart(_localKey));
+	}
+	_owner->accountAddedInStorage(Main::Domain::AccountWithIndex{
+		.account = std::move(account)
+	});
+	writeAccounts();
+}
+
+void Domain::generateLocalKey() {
+	Expects(_localKey == nullptr);
+	Expects(_passcodeKeySalt.isEmpty());
+	Expects(_passcodeKeyEncrypted.isEmpty());
+
+	auto pass = QByteArray(MTP::AuthKey::kSize, Qt::Uninitialized);
+	auto salt = QByteArray(LocalEncryptSaltSize, Qt::Uninitialized);
+	base::RandomFill(pass.data(), pass.size());
+	base::RandomFill(salt.data(), salt.size());
+	_localKey = CreateLocalKey(pass, salt);
+
+	encryptLocalKey(QByteArray());
+}
+
+void Domain::encryptLocalKey(const QByteArray &passcode) {
+	_passcodeKeySalt.resize(LocalEncryptSaltSize);
+	base::RandomFill(_passcodeKeySalt.data(), _passcodeKeySalt.size());
+	_passcodeKey = CreateLocalKey(passcode, _passcodeKeySalt);
+
+	EncryptedDescriptor passKeyData(MTP::AuthKey::kSize);
+	_localKey->write(passKeyData.stream);
+	_passcodeKeyEncrypted = PrepareEncrypted(passKeyData, _passcodeKey);
+	_hasLocalPasscode = !passcode.isEmpty();
+}
+
+Domain::StartModernResult Domain::startModern(
+		const QByteArray &passcode) {
+	const auto name = ComputeKeyName(_dataName);
+
+	FileReadDescriptor keyData;
+	if (!ReadFile(keyData, name, BaseGlobalPath())) {
+		return StartModernResult::Empty;
+	}
+	LOG(("App Info: reading accounts info..."));
+
+	QByteArray salt, keyEncrypted, infoEncrypted;
+	keyData.stream >> salt >> keyEncrypted >> infoEncrypted;
+	if (!CheckStreamStatus(keyData.stream)) {
+		return StartModernResult::Failed;
+	}
+
+	if (salt.size() != LocalEncryptSaltSize) {
+		LOG(("App Error: bad salt in info file, size: %1").arg(salt.size()));
+		return StartModernResult::Failed;
+	}
+	_passcodeKey = CreateLocalKey(passcode, salt);
+
+	EncryptedDescriptor keyInnerData, info;
+	if (!DecryptLocal(keyInnerData, keyEncrypted, _passcodeKey)) {
+		LOG(("App Info: could not decrypt pass-protected key from info file, "
+			"maybe bad password..."));
+		return StartModernResult::IncorrectPasscode;
+	}
+	auto key = Serialize::read<MTP::AuthKey::Data>(keyInnerData.stream);
+	if (keyInnerData.stream.status() != QDataStream::Ok
+		|| !keyInnerData.stream.atEnd()) {
+		LOG(("App Error: could not read pass-protected key from info file"));
+		return StartModernResult::Failed;
+	}
+	_localKey = std::make_shared<MTP::AuthKey>(key);
+
+	_passcodeKeyEncrypted = keyEncrypted;
+	_passcodeKeySalt = salt;
+	_hasLocalPasscode = !passcode.isEmpty();
+
+	if (!DecryptLocal(info, infoEncrypted, _localKey)) {
+		LOG(("App Error: could not decrypt info."));
+		return StartModernResult::Failed;
+	}
+	LOG(("App Info: reading encrypted info..."));
+	auto count = qint32();
+	info.stream >> count;
+	if (count <= 0 || count > Main::Domain::kPremiumMaxAccounts) {
+		LOG(("App Error: bad accounts count: %1").arg(count));
+		return StartModernResult::Failed;
+	}
+
+	_oldVersion = keyData.version;
+
+	auto tried = base::flat_set<int>();
+	auto sessions = base::flat_set<uint64>();
+	auto active = 0;
+	for (auto i = 0; i != count; ++i) {
+		auto index = qint32();
+		info.stream >> index;
+		if (index >= 0
+			&& index < Main::Domain::kPremiumMaxAccounts
+			&& tried.emplace(index).second) {
+			auto account = std::make_unique<Main::Account>(
+				_owner,
+				_dataName,
+				index);
+			auto config = account->prepareToStart(_localKey);
+			const auto sessionId = account->willHaveSessionUniqueId(
+				config.get());
+			if (!sessions.contains(sessionId)
+				&& (sessionId != 0 || (sessions.empty() && i + 1 == count))) {
+				if (sessions.empty()) {
+					active = index;
+				}
+				account->start(std::move(config));
+				_owner->accountAddedInStorage({
+					.index = index,
+					.account = std::move(account)
+				});
+				sessions.emplace(sessionId);
+			}
+		}
+	}
+	if (sessions.empty()) {
+		LOG(("App Error: no accounts read."));
+		return StartModernResult::Failed;
+	}
+
+	if (!info.stream.atEnd()) {
+		info.stream >> active;
+	}
+	_owner->activateFromStorage(active);
+
+	Ensures(!sessions.empty());
+	return StartModernResult::Success;
+}
+
+void Domain::writeAccounts() {
+	Expects(!_owner->accounts().empty());
+
+	const auto path = BaseGlobalPath();
+	if (!QDir().exists(path)) {
+		QDir().mkpath(path);
+	}
+
+	FileWriteDescriptor key(ComputeKeyName(_dataName), path);
+	key.writeData(_passcodeKeySalt);
+	key.writeData(_passcodeKeyEncrypted);
+
+	const auto &list = _owner->accounts();
+
+	auto keySize = sizeof(qint32) + sizeof(qint32) * list.size();
+
+	EncryptedDescriptor keyData(keySize);
+	keyData.stream << qint32(list.size());
+	for (const auto &[index, account] : list) {
+		keyData.stream << qint32(index);
+	}
+	keyData.stream << qint32(_owner->activeForStorage());
+	key.writeEncrypted(keyData, _localKey);
+}
+
+void Domain::startFromScratch() {
+	startWithSingleAccount(
+		QByteArray(),
+		std::make_unique<Main::Account>(_owner, _dataName, 0));
+}
+
+bool Domain::checkPasscode(const QByteArray &passcode) const {
+	Expects(!_passcodeKeySalt.isEmpty());
+	Expects(_passcodeKey != nullptr);
+
+	const auto checkKey = CreateLocalKey(passcode, _passcodeKeySalt);
+	return checkKey->equals(_passcodeKey);
+}
+
+void Domain::setPasscode(const QByteArray &passcode) {
+	Expects(!_passcodeKeySalt.isEmpty());
+	Expects(_localKey != nullptr);
+
+	encryptLocalKey(passcode);
+	writeAccounts();
+
+	_passcodeKeyChanged.fire({});
+}
+
+int Domain::oldVersion() const {
+	return _oldVersion;
+}
+
+void Domain::clearOldVersion() {
+	_oldVersion = 0;
+}
+
+rpl::producer<> Domain::localPasscodeChanged() const {
+	return _passcodeKeyChanged.events();
+}
+
+bool Domain::hasLocalPasscode() const {
+	return _hasLocalPasscode;
+}
+
+} // namespace Storage

Telegram/SourceFiles/storage/storage_domain.h

@@ -0,0 +1,79 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#pragma once
+
+namespace MTP {
+class Config;
+class AuthKey;
+using AuthKeyPtr = std::shared_ptr<AuthKey>;
+} // namespace MTP
+
+namespace Main {
+class Account;
+class Domain;
+} // namespace Main
+
+namespace Storage {
+
+enum class StartResult : uchar {
+	Success,
+	IncorrectPasscode,
+	IncorrectPasscodeLegacy,
+};
+
+class Domain final {
+public:
+	Domain(not_null<Main::Domain*> owner, const QString &dataName);
+	~Domain();
+
+	[[nodiscard]] StartResult start(const QByteArray &passcode);
+	void startAdded(
+		not_null<Main::Account*> account,
+		std::unique_ptr<MTP::Config> config);
+	void writeAccounts();
+	void startFromScratch();
+
+	[[nodiscard]] bool checkPasscode(const QByteArray &passcode) const;
+	void setPasscode(const QByteArray &passcode);
+
+	[[nodiscard]] int oldVersion() const;
+	void clearOldVersion();
+
+	[[nodiscard]] rpl::producer<> localPasscodeChanged() const;
+	[[nodiscard]] bool hasLocalPasscode() const;
+
+private:
+	enum class StartModernResult {
+		Success,
+		IncorrectPasscode,
+		Failed,
+		Empty,
+	};
+
+	[[nodiscard]] StartModernResult startModern(const QByteArray &passcode);
+	void startWithSingleAccount(
+		const QByteArray &passcode,
+		std::unique_ptr<Main::Account> account);
+	void generateLocalKey();
+	void encryptLocalKey(const QByteArray &passcode);
+
+	const not_null<Main::Domain*> _owner;
+	const QString _dataName;
+
+	MTP::AuthKeyPtr _localKey;
+	MTP::AuthKeyPtr _passcodeKey;
+	QByteArray _passcodeKeySalt;
+	QByteArray _passcodeKeyEncrypted;
+	int _oldVersion = 0;
+
+	bool _hasLocalPasscode = false;
+	rpl::event_stream<> _passcodeKeyChanged;
+
+};
+
+} // namespace Storage

Telegram/SourceFiles/storage/storage_facade.cpp

@@ -0,0 +1,228 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#include "storage/storage_facade.h"
+
+#include "storage/storage_shared_media.h"
+#include "storage/storage_user_photos.h"
+
+namespace Storage {
+
+class Facade::Impl {
+public:
+	void add(SharedMediaAddNew &&query);
+	void add(SharedMediaAddExisting &&query);
+	void add(SharedMediaAddSlice &&query);
+	void remove(SharedMediaRemoveOne &&query);
+	void remove(SharedMediaRemoveAll &&query);
+	void invalidate(SharedMediaInvalidateBottom &&query);
+	void unload(SharedMediaUnloadThread &&query);
+	rpl::producer<SharedMediaResult> query(SharedMediaQuery &&query) const;
+	SharedMediaResult snapshot(const SharedMediaQuery &query) const;
+	bool empty(const SharedMediaKey &key) const;
+	rpl::producer<SharedMediaSliceUpdate> sharedMediaSliceUpdated() const;
+	rpl::producer<SharedMediaRemoveOne> sharedMediaOneRemoved() const;
+	rpl::producer<SharedMediaRemoveAll> sharedMediaAllRemoved() const;
+	rpl::producer<SharedMediaInvalidateBottom> sharedMediaBottomInvalidated() const;
+
+	void add(UserPhotosSetBack &&query);
+	void add(UserPhotosAddNew &&query);
+	void add(UserPhotosAddSlice &&query);
+	void remove(UserPhotosRemoveOne &&query);
+	void remove(UserPhotosRemoveAfter &&query);
+	void replace(UserPhotosReplace &&query);
+	rpl::producer<UserPhotosResult> query(UserPhotosQuery &&query) const;
+	rpl::producer<UserPhotosSliceUpdate> userPhotosSliceUpdated() const;
+
+private:
+	SharedMedia _sharedMedia;
+	UserPhotos _userPhotos;
+
+};
+
+void Facade::Impl::add(SharedMediaAddNew &&query) {
+	_sharedMedia.add(std::move(query));
+}
+
+void Facade::Impl::add(SharedMediaAddExisting &&query) {
+	_sharedMedia.add(std::move(query));
+}
+
+void Facade::Impl::add(SharedMediaAddSlice &&query) {
+	_sharedMedia.add(std::move(query));
+}
+
+void Facade::Impl::remove(SharedMediaRemoveOne &&query) {
+	_sharedMedia.remove(std::move(query));
+}
+
+void Facade::Impl::remove(SharedMediaRemoveAll &&query) {
+	_sharedMedia.remove(std::move(query));
+}
+
+void Facade::Impl::invalidate(SharedMediaInvalidateBottom &&query) {
+	_sharedMedia.invalidate(std::move(query));
+}
+
+void Facade::Impl::unload(SharedMediaUnloadThread &&query) {
+	_sharedMedia.unload(std::move(query));
+}
+
+rpl::producer<SharedMediaResult> Facade::Impl::query(SharedMediaQuery &&query) const {
+	return _sharedMedia.query(std::move(query));
+}
+
+SharedMediaResult Facade::Impl::snapshot(const SharedMediaQuery &query) const {
+	return _sharedMedia.snapshot(query);
+}
+
+bool Facade::Impl::empty(const SharedMediaKey &key) const {
+	return _sharedMedia.empty(key);
+}
+
+rpl::producer<SharedMediaSliceUpdate> Facade::Impl::sharedMediaSliceUpdated() const {
+	return _sharedMedia.sliceUpdated();
+}
+
+rpl::producer<SharedMediaRemoveOne> Facade::Impl::sharedMediaOneRemoved() const {
+	return _sharedMedia.oneRemoved();
+}
+
+rpl::producer<SharedMediaRemoveAll> Facade::Impl::sharedMediaAllRemoved() const {
+	return _sharedMedia.allRemoved();
+}
+
+rpl::producer<SharedMediaInvalidateBottom> Facade::Impl::sharedMediaBottomInvalidated() const {
+	return _sharedMedia.bottomInvalidated();
+}
+
+void Facade::Impl::add(UserPhotosSetBack &&query) {
+	return _userPhotos.add(std::move(query));
+}
+
+void Facade::Impl::add(UserPhotosAddNew &&query) {
+	return _userPhotos.add(std::move(query));
+}
+
+void Facade::Impl::add(UserPhotosAddSlice &&query) {
+	return _userPhotos.add(std::move(query));
+}
+
+void Facade::Impl::remove(UserPhotosRemoveOne &&query) {
+	return _userPhotos.remove(std::move(query));
+}
+
+void Facade::Impl::remove(UserPhotosRemoveAfter &&query) {
+	return _userPhotos.remove(std::move(query));
+}
+
+void Facade::Impl::replace(UserPhotosReplace &&query) {
+	return _userPhotos.replace(std::move(query));
+}
+
+rpl::producer<UserPhotosResult> Facade::Impl::query(UserPhotosQuery &&query) const {
+	return _userPhotos.query(std::move(query));
+}
+
+rpl::producer<UserPhotosSliceUpdate> Facade::Impl::userPhotosSliceUpdated() const {
+	return _userPhotos.sliceUpdated();
+}
+
+Facade::Facade() : _impl(std::make_unique<Impl>()) {
+}
+
+void Facade::add(SharedMediaAddNew &&query) {
+	_impl->add(std::move(query));
+}
+
+void Facade::add(SharedMediaAddExisting &&query) {
+	_impl->add(std::move(query));
+}
+
+void Facade::add(SharedMediaAddSlice &&query) {
+	_impl->add(std::move(query));
+}
+
+void Facade::remove(SharedMediaRemoveOne &&query) {
+	_impl->remove(std::move(query));
+}
+
+void Facade::remove(SharedMediaRemoveAll &&query) {
+	_impl->remove(std::move(query));
+}
+
+void Facade::invalidate(SharedMediaInvalidateBottom &&query) {
+	_impl->invalidate(std::move(query));
+}
+
+void Facade::unload(SharedMediaUnloadThread &&query) {
+	_impl->unload(std::move(query));
+}
+
+rpl::producer<SharedMediaResult> Facade::query(SharedMediaQuery &&query) const {
+	return _impl->query(std::move(query));
+}
+
+SharedMediaResult Facade::snapshot(const SharedMediaQuery &query) const {
+	return _impl->snapshot(query);
+}
+
+bool Facade::empty(const SharedMediaKey &key) const {
+	return _impl->empty(key);
+}
+
+rpl::producer<SharedMediaSliceUpdate> Facade::sharedMediaSliceUpdated() const {
+	return _impl->sharedMediaSliceUpdated();
+}
+
+rpl::producer<SharedMediaRemoveOne> Facade::sharedMediaOneRemoved() const {
+	return _impl->sharedMediaOneRemoved();
+}
+
+rpl::producer<SharedMediaRemoveAll> Facade::sharedMediaAllRemoved() const {
+	return _impl->sharedMediaAllRemoved();
+}
+
+rpl::producer<SharedMediaInvalidateBottom> Facade::sharedMediaBottomInvalidated() const {
+	return _impl->sharedMediaBottomInvalidated();
+}
+
+void Facade::add(UserPhotosSetBack &&query) {
+	return _impl->add(std::move(query));
+}
+
+void Facade::add(UserPhotosAddNew &&query) {
+	return _impl->add(std::move(query));
+}
+
+void Facade::add(UserPhotosAddSlice &&query) {
+	return _impl->add(std::move(query));
+}
+
+void Facade::remove(UserPhotosRemoveOne &&query) {
+	return _impl->remove(std::move(query));
+}
+
+void Facade::remove(UserPhotosRemoveAfter &&query) {
+	return _impl->remove(std::move(query));
+}
+
+void Facade::replace(UserPhotosReplace &&query) {
+	return _impl->replace(std::move(query));
+}
+
+rpl::producer<UserPhotosResult> Facade::query(UserPhotosQuery &&query) const {
+	return _impl->query(std::move(query));
+}
+
+rpl::producer<UserPhotosSliceUpdate> Facade::userPhotosSliceUpdated() const {
+	return _impl->userPhotosSliceUpdated();
+}
+
+Facade::~Facade() = default;
+
+} // namespace Storage

Telegram/SourceFiles/storage/storage_facade.h

@@ -0,0 +1,81 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#pragma once
+
+#include <rpl/producer.h>
+#include "base/enum_mask.h"
+
+namespace Data {
+struct MessagesResult;
+} // namespace Data
+
+namespace Storage {
+
+struct SparseIdsListResult;
+
+struct SharedMediaAddNew;
+struct SharedMediaAddExisting;
+struct SharedMediaAddSlice;
+struct SharedMediaRemoveOne;
+struct SharedMediaRemoveAll;
+struct SharedMediaInvalidateBottom;
+struct SharedMediaUnloadThread;
+struct SharedMediaQuery;
+struct SharedMediaKey;
+using SharedMediaResult = SparseIdsListResult;
+struct SharedMediaSliceUpdate;
+
+struct UserPhotosSetBack;
+struct UserPhotosAddNew;
+struct UserPhotosAddSlice;
+struct UserPhotosRemoveOne;
+struct UserPhotosRemoveAfter;
+struct UserPhotosReplace;
+struct UserPhotosQuery;
+struct UserPhotosResult;
+struct UserPhotosSliceUpdate;
+
+class Facade {
+public:
+	Facade();
+
+	void add(SharedMediaAddNew &&query);
+	void add(SharedMediaAddExisting &&query);
+	void add(SharedMediaAddSlice &&query);
+	void remove(SharedMediaRemoveOne &&query);
+	void remove(SharedMediaRemoveAll &&query);
+	void invalidate(SharedMediaInvalidateBottom &&query);
+	void unload(SharedMediaUnloadThread &&query);
+
+	rpl::producer<SharedMediaResult> query(SharedMediaQuery &&query) const;
+	SharedMediaResult snapshot(const SharedMediaQuery &query) const;
+	bool empty(const SharedMediaKey &key) const;
+	rpl::producer<SharedMediaSliceUpdate> sharedMediaSliceUpdated() const;
+	rpl::producer<SharedMediaRemoveOne> sharedMediaOneRemoved() const;
+	rpl::producer<SharedMediaRemoveAll> sharedMediaAllRemoved() const;
+	rpl::producer<SharedMediaInvalidateBottom> sharedMediaBottomInvalidated() const;
+
+	void add(UserPhotosSetBack &&query);
+	void add(UserPhotosAddNew &&query);
+	void add(UserPhotosAddSlice &&query);
+	void remove(UserPhotosRemoveOne &&query);
+	void remove(UserPhotosRemoveAfter &&query);
+	void replace(UserPhotosReplace &&query);
+
+	rpl::producer<UserPhotosResult> query(UserPhotosQuery &&query) const;
+	rpl::producer<UserPhotosSliceUpdate> userPhotosSliceUpdated() const;
+
+	~Facade();
+
+private:
+	class Impl;
+	const std::unique_ptr<Impl> _impl;
+
+};
+
+} // namespace Storage

Telegram/SourceFiles/storage/storage_file_lock.h

@@ -0,0 +1,38 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#pragma once
+
+#include "base/basic_types.h"
+#include <QtCore/QFile>
+
+namespace Storage {
+
+class FileLock {
+public:
+	FileLock();
+
+	bool lock(QFile &file, QIODevice::OpenMode mode);
+	void unlock();
+
+	static constexpr auto kSkipBytes = size_type(4);
+
+	~FileLock();
+
+private:
+	class Lock;
+	struct Descriptor;
+	struct LockingPid;
+
+	static constexpr auto kLockOffset = index_type(0);
+	static constexpr auto kLockLimit = kSkipBytes;
+
+	std::unique_ptr<Lock> _lock;
+
+};
+
+} // namespace Storage

Telegram/SourceFiles/storage/storage_file_lock_posix.cpp

@@ -0,0 +1,124 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#include "storage/storage_file_lock.h"
+
+#include "base/variant.h"
+
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/types.h>
+#include <signal.h>
+
+namespace Storage {
+namespace {
+
+bool KillProcess(pid_t pid) {
+	auto signal = SIGTERM;
+	auto attempts = 0;
+	while (true) {
+		const auto result = kill(pid, signal);
+		if (result < 0) {
+			return (errno == ESRCH);
+		}
+		usleep(10000);
+		if (++attempts == 50) {
+			signal = SIGKILL;
+		}
+	}
+}
+
+} // namespace
+
+struct FileLock::Descriptor {
+	int value;
+};
+
+struct FileLock::LockingPid {
+	pid_t value;
+};
+
+class FileLock::Lock {
+public:
+	using Result = base::variant<Descriptor, LockingPid>;
+	static Result Acquire(const QFile &file);
+
+	explicit Lock(int descriptor);
+	~Lock();
+
+private:
+	int _descriptor = 0;
+
+};
+
+FileLock::Lock::Result FileLock::Lock::Acquire(const QFile &file) {
+	const auto descriptor = file.handle();
+	if (!descriptor || !file.isOpen()) {
+		return Descriptor{ 0 };
+	}
+	while (true) {
+		struct flock lock;
+		lock.l_type = F_WRLCK;
+		lock.l_whence = SEEK_SET;
+		lock.l_start = kLockOffset;
+		lock.l_len = kLockLimit;
+		if (fcntl(descriptor, F_SETLK, &lock) == 0) {
+			return Descriptor{ descriptor };
+		} else if (fcntl(descriptor, F_GETLK, &lock) < 0) {
+			return LockingPid{ 0 };
+		} else if (lock.l_type != F_UNLCK) {
+			return LockingPid{ lock.l_pid };
+		}
+	}
+}
+
+FileLock::Lock::Lock(int descriptor) : _descriptor(descriptor) {
+}
+
+FileLock::Lock::~Lock() {
+	struct flock unlock;
+	unlock.l_type = F_UNLCK;
+	unlock.l_whence = SEEK_SET;
+	unlock.l_start = kLockOffset;
+	unlock.l_len = kLockLimit;
+	fcntl(_descriptor, F_SETLK, &unlock);
+}
+
+FileLock::FileLock() = default;
+
+bool FileLock::lock(QFile &file, QIODevice::OpenMode mode) {
+	Expects(_lock == nullptr || file.isOpen());
+
+	unlock();
+	file.close();
+	if (!file.open(mode)) {
+		return false;
+	}
+	while (true) {
+		const auto result = Lock::Acquire(file);
+		if (const auto descriptor = base::get_if<Descriptor>(&result)) {
+			if (descriptor->value > 0) {
+				_lock = std::make_unique<Lock>(descriptor->value);
+				return true;
+			}
+			break;
+		} else if (const auto pid = base::get_if<LockingPid>(&result)) {
+			if (pid->value <= 0 || !KillProcess(pid->value)) {
+				break;
+			}
+		}
+	}
+	return false;
+}
+
+void FileLock::unlock() {
+	_lock = nullptr;
+}
+
+FileLock::~FileLock() = default;
+
+} // namespace Storage

Telegram/SourceFiles/storage/storage_file_lock_win.cpp

@@ -0,0 +1,143 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#include "storage/storage_file_lock.h"
+
+#include "platform/win/windows_dlls.h"
+#include "base/platform/win/base_windows_h.h"
+
+#include <io.h>
+#include <fileapi.h>
+#include <RestartManager.h>
+
+namespace Storage {
+namespace {
+
+bool CloseProcesses(const QString &filename) {
+	using namespace Platform;
+
+	if (!Dlls::RmStartSession
+		|| !Dlls::RmRegisterResources
+		|| !Dlls::RmGetList
+		|| !Dlls::RmShutdown
+		|| !Dlls::RmEndSession) {
+		return false;
+	}
+	auto result = BOOL(FALSE);
+	auto session = DWORD();
+	auto sessionKey = std::wstring(CCH_RM_SESSION_KEY + 1, wchar_t(0));
+	auto error = Dlls::RmStartSession(&session, 0, sessionKey.data());
+	if (error != ERROR_SUCCESS) {
+		return false;
+	}
+	const auto guard = gsl::finally([&] { Dlls::RmEndSession(session); });
+
+	const auto path = QDir::toNativeSeparators(filename).toStdWString();
+	auto nullterm = path.c_str();
+	error = Dlls::RmRegisterResources(
+		session,
+		1,
+		&nullterm,
+		0,
+		nullptr,
+		0,
+		nullptr);
+	if (error != ERROR_SUCCESS) {
+		return false;
+	}
+
+	auto processInfoNeeded = UINT(0);
+	auto processInfoCount = UINT(0);
+	auto reason = DWORD();
+
+	error = Dlls::RmGetList(
+		session,
+		&processInfoNeeded,
+		&processInfoCount,
+		nullptr,
+		&reason);
+	if (error != ERROR_SUCCESS && error != ERROR_MORE_DATA) {
+		return false;
+	} else if (processInfoNeeded <= 0) {
+		return true;
+	}
+	error = Dlls::RmShutdown(session, RmForceShutdown, NULL);
+	if (error != ERROR_SUCCESS) {
+		return false;
+	}
+	return true;
+}
+
+} // namespace
+
+class FileLock::Lock {
+public:
+	static int Acquire(const QFile &file);
+
+	explicit Lock(int descriptor);
+	~Lock();
+
+private:
+	static constexpr auto offsetLow = DWORD(kLockOffset);
+	static constexpr auto offsetHigh = DWORD(0);
+	static constexpr auto limitLow = DWORD(kLockLimit);
+	static constexpr auto limitHigh = DWORD(0);
+
+	int _descriptor = 0;
+
+};
+
+int FileLock::Lock::Acquire(const QFile &file) {
+	const auto descriptor = file.handle();
+	if (!descriptor || !file.isOpen()) {
+		return false;
+	}
+	const auto handle = HANDLE(_get_osfhandle(descriptor));
+	if (!handle) {
+		return false;
+	}
+	return LockFile(handle, offsetLow, offsetHigh, limitLow, limitHigh)
+		? descriptor
+		: 0;
+}
+
+FileLock::Lock::Lock(int descriptor) : _descriptor(descriptor) {
+}
+
+FileLock::Lock::~Lock() {
+	if (const auto handle = HANDLE(_get_osfhandle(_descriptor))) {
+		UnlockFile(handle, offsetLow, offsetHigh, limitLow, limitHigh);
+	}
+}
+
+FileLock::FileLock() = default;
+
+bool FileLock::lock(QFile &file, QIODevice::OpenMode mode) {
+	Expects(_lock == nullptr || file.isOpen());
+
+	unlock();
+	file.close();
+	do {
+		if (!file.open(mode)) {
+			return false;
+		} else if (const auto descriptor = Lock::Acquire(file)) {
+			_lock = std::make_unique<Lock>(descriptor);
+			return true;
+		}
+		file.close();
+	} while (CloseProcesses(file.fileName()));
+
+	return false;
+}
+
+void FileLock::unlock() {
+	_lock = nullptr;
+}
+
+FileLock::~FileLock() = default;
+
+} // namespace Storage

Telegram/SourceFiles/storage/storage_media_prepare.cpp

@@ -0,0 +1,403 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#include "storage/storage_media_prepare.h"
+
+#include "editor/photo_editor_common.h"
+#include "platform/platform_file_utilities.h"
+#include "lang/lang_keys.h"
+#include "storage/localimageloader.h"
+#include "core/mime_type.h"
+#include "ui/image/image_prepare.h"
+#include "ui/chat/attach/attach_prepare.h"
+#include "core/crash_reports.h"
+
+#include <QtCore/QSemaphore>
+#include <QtCore/QMimeData>
+
+namespace Storage {
+namespace {
+
+using Ui::PreparedFileInformation;
+using Ui::PreparedFile;
+using Ui::PreparedList;
+
+using Image = PreparedFileInformation::Image;
+
+bool ValidPhotoForAlbum(
+		const Image &image,
+		const QString &mime) {
+	Expects(!image.data.isNull());
+
+	if (image.animated
+		|| (!mime.isEmpty() && !mime.startsWith(u"image/"))) {
+		return false;
+	}
+	const auto width = image.data.width();
+	const auto height = image.data.height();
+	return Ui::ValidateThumbDimensions(width, height);
+}
+
+bool ValidVideoForAlbum(const PreparedFileInformation::Video &video) {
+	const auto width = video.thumbnail.width();
+	const auto height = video.thumbnail.height();
+	return Ui::ValidateThumbDimensions(width, height);
+}
+
+QSize PrepareShownDimensions(const QImage &preview, int sideLimit) {
+	const auto result = preview.size();
+	return (result.width() > sideLimit || result.height() > sideLimit)
+		? result.scaled(sideLimit, sideLimit, Qt::KeepAspectRatio)
+		: result;
+}
+
+void PrepareDetailsInParallel(PreparedList &result, int previewWidth) {
+	Expects(result.files.size() <= Ui::MaxAlbumItems());
+
+	if (result.files.empty()) {
+		return;
+	}
+	const auto sideLimit = PhotoSideLimit(); // Get on main thread.
+	QSemaphore semaphore;
+	for (auto &file : result.files) {
+		crl::async([=, &semaphore, &file] {
+			PrepareDetails(file, previewWidth, sideLimit);
+			semaphore.release();
+		});
+	}
+	semaphore.acquire(result.files.size());
+}
+
+} // namespace
+
+bool ValidatePhotoEditorMediaDragData(not_null<const QMimeData*> data) {
+	const auto urls = Core::ReadMimeUrls(data);
+	if (urls.size() > 1) {
+		return false;
+	} else if (data->hasImage()) {
+		return true;
+	}
+
+	if (!urls.isEmpty()) {
+		const auto url = urls.front();
+		if (url.isLocalFile()) {
+			using namespace Core;
+			const auto file = Platform::File::UrlToLocal(url);
+			const auto info = QFileInfo(file);
+			return FileIsImage(file, MimeTypeForFile(info).name())
+				&& QImageReader(file).canRead();
+		}
+	}
+
+	return false;
+}
+
+bool ValidateEditMediaDragData(
+		not_null<const QMimeData*> data,
+		Ui::AlbumType albumType) {
+	const auto urls = Core::ReadMimeUrls(data);
+	if (urls.size() > 1) {
+		return false;
+	} else if (data->hasImage()) {
+		return (albumType != Ui::AlbumType::Music);
+	}
+
+	if (albumType == Ui::AlbumType::PhotoVideo && !urls.isEmpty()) {
+		const auto url = urls.front();
+		if (url.isLocalFile()) {
+			using namespace Core;
+			const auto info = QFileInfo(Platform::File::UrlToLocal(url));
+			return IsMimeAcceptedForPhotoVideoAlbum(MimeTypeForFile(info).name());
+		}
+	}
+
+	return true;
+}
+
+MimeDataState ComputeMimeDataState(const QMimeData *data) {
+	if (!data || data->hasFormat(u"application/x-td-forward"_q)) {
+		return MimeDataState::None;
+	}
+
+	if (data->hasImage()) {
+		return MimeDataState::Image;
+	}
+
+	const auto urls = Core::ReadMimeUrls(data);
+	if (urls.isEmpty()) {
+		return MimeDataState::None;
+	}
+
+	auto allAreSmallImages = true;
+	for (const auto &url : urls) {
+		if (!url.isLocalFile()) {
+			return MimeDataState::None;
+		}
+		const auto file = Platform::File::UrlToLocal(url);
+
+		const auto info = QFileInfo(file);
+		if (info.isDir()) {
+			return MimeDataState::None;
+		}
+
+		using namespace Core;
+		const auto filesize = info.size();
+		if (filesize > kFileSizePremiumLimit) {
+			return MimeDataState::None;
+		//} else if (filesize > kFileSizeLimit) {
+		//	return MimeDataState::PremiumFile;
+		} else if (allAreSmallImages) {
+			if (filesize > Images::kReadBytesLimit) {
+				allAreSmallImages = false;
+			} else {
+				const auto mime = MimeTypeForFile(info).name();
+				if (mime == u"image/gif"_q
+					|| !FileIsImage(file, mime)
+					|| !QImageReader(file).canRead()) {
+					allAreSmallImages = false;
+				}
+			}
+		}
+	}
+	return allAreSmallImages
+		? MimeDataState::PhotoFiles
+		: MimeDataState::Files;
+}
+
+PreparedList PrepareMediaList(
+		const QList<QUrl> &files,
+		int previewWidth,
+		bool premium) {
+	auto locals = QStringList();
+	locals.reserve(files.size());
+	for (const auto &url : files) {
+		if (!url.isLocalFile()) {
+			return {
+				PreparedList::Error::NonLocalUrl,
+				url.toDisplayString()
+			};
+		}
+		locals.push_back(Platform::File::UrlToLocal(url));
+	}
+	return PrepareMediaList(locals, previewWidth, premium);
+}
+
+PreparedList PrepareMediaList(
+		const QStringList &files,
+		int previewWidth,
+		bool premium) {
+	auto result = PreparedList();
+	result.files.reserve(files.size());
+	for (const auto &file : files) {
+		const auto fileinfo = QFileInfo(file);
+		const auto filesize = fileinfo.size();
+		if (fileinfo.isDir()) {
+			return {
+				PreparedList::Error::Directory,
+				file
+			};
+		} else if (filesize <= 0) {
+			return {
+				PreparedList::Error::EmptyFile,
+				file
+			};
+		} else if (filesize > kFileSizePremiumLimit
+			|| (filesize > kFileSizeLimit && !premium)) {
+			auto errorResult = PreparedList(
+				PreparedList::Error::TooLargeFile,
+				QString());
+			errorResult.files.emplace_back(file);
+			errorResult.files.back().size = filesize;
+			return errorResult;
+		}
+		if (result.files.size() < Ui::MaxAlbumItems()) {
+			result.files.emplace_back(file);
+			result.files.back().size = filesize;
+		} else {
+			result.filesToProcess.emplace_back(file);
+			result.files.back().size = filesize;
+		}
+	}
+	PrepareDetailsInParallel(result, previewWidth);
+	return result;
+}
+
+PreparedList PrepareMediaFromImage(
+		QImage &&image,
+		QByteArray &&content,
+		int previewWidth) {
+	Expects(!image.isNull());
+
+	auto result = PreparedList();
+	auto file = PreparedFile(QString());
+	file.content = content;
+	if (file.content.isEmpty()) {
+		file.information = std::make_unique<PreparedFileInformation>();
+		const auto animated = false;
+		FileLoadTask::FillImageInformation(
+			std::move(image),
+			animated,
+			file.information);
+	}
+	result.files.push_back(std::move(file));
+	PrepareDetailsInParallel(result, previewWidth);
+	return result;
+}
+
+std::optional<PreparedList> PreparedFileFromFilesDialog(
+		FileDialog::OpenResult &&result,
+		Fn<bool(const Ui::PreparedList&)> checkResult,
+		Fn<void(tr::phrase<>)> errorCallback,
+		int previewWidth,
+		bool premium) {
+	if (result.paths.isEmpty() && result.remoteContent.isEmpty()) {
+		return std::nullopt;
+	}
+
+	auto list = result.remoteContent.isEmpty()
+		? PrepareMediaList(result.paths, previewWidth, premium)
+		: PrepareMediaFromImage(
+			QImage(),
+			std::move(result.remoteContent),
+			previewWidth);
+	if (list.error != PreparedList::Error::None) {
+		errorCallback(tr::lng_send_media_invalid_files);
+		return std::nullopt;
+	} else if (!checkResult(list)) {
+		return std::nullopt;
+	} else {
+		return list;
+	}
+}
+
+void PrepareDetails(PreparedFile &file, int previewWidth, int sideLimit) {
+	if (!file.path.isEmpty()) {
+		file.information = FileLoadTask::ReadMediaInformation(
+			file.path,
+			QByteArray(),
+			Core::MimeTypeForFile(QFileInfo(file.path)).name());
+	} else if (!file.content.isEmpty()) {
+		file.information = FileLoadTask::ReadMediaInformation(
+			QString(),
+			file.content,
+			Core::MimeTypeForData(file.content).name());
+	} else {
+		Assert(file.information != nullptr);
+	}
+
+	using Video = PreparedFileInformation::Video;
+	using Song = PreparedFileInformation::Song;
+	if (const auto image = std::get_if<Image>(
+			&file.information->media)) {
+		Assert(!image->data.isNull());
+		if (ValidPhotoForAlbum(*image, file.information->filemime)) {
+			UpdateImageDetails(file, previewWidth, sideLimit);
+			file.type = PreparedFile::Type::Photo;
+		} else {
+			file.originalDimensions = image->data.size();
+			if (image->animated) {
+				file.type = PreparedFile::Type::None;
+			}
+		}
+	} else if (const auto video = std::get_if<Video>(
+			&file.information->media)) {
+		if (ValidVideoForAlbum(*video)) {
+			auto blurred = Images::Blur(
+				Images::Opaque(base::duplicate(video->thumbnail)));
+			file.originalDimensions = video->thumbnail.size();
+			file.shownDimensions = PrepareShownDimensions(
+				video->thumbnail,
+				sideLimit);
+			file.preview = std::move(blurred).scaledToWidth(
+				previewWidth * style::DevicePixelRatio(),
+				Qt::SmoothTransformation);
+			Assert(!file.preview.isNull());
+			file.preview.setDevicePixelRatio(style::DevicePixelRatio());
+			file.type = PreparedFile::Type::Video;
+		}
+	} else if (v::is<Song>(file.information->media)) {
+		file.type = PreparedFile::Type::Music;
+	}
+}
+
+void UpdateImageDetails(
+		PreparedFile &file,
+		int previewWidth,
+		int sideLimit) {
+	const auto image = std::get_if<Image>(&file.information->media);
+	if (!image) {
+		return;
+	}
+	Assert(!image->data.isNull());
+	auto preview = image->modifications
+		? Editor::ImageModified(image->data, image->modifications)
+		: image->data;
+	Assert(!preview.isNull());
+	file.originalDimensions = preview.size();
+	file.shownDimensions = PrepareShownDimensions(preview, sideLimit);
+	const auto toWidth = std::min(
+		previewWidth,
+		style::ConvertScale(preview.width())
+	) * style::DevicePixelRatio();
+	auto scaled = preview.scaledToWidth(
+		toWidth,
+		Qt::SmoothTransformation);
+	if (scaled.isNull()) {
+		CrashReports::SetAnnotation("Info", QString("%1x%2:%3*%4->%5;%6x%7"
+		).arg(preview.width()).arg(preview.height()
+		).arg(previewWidth).arg(style::DevicePixelRatio()
+		).arg(toWidth
+		).arg(scaled.width()).arg(scaled.height()));
+		Unexpected("Scaled is null.");
+	}
+	Assert(!scaled.isNull());
+	file.preview = Images::Opaque(std::move(scaled));
+	Assert(!file.preview.isNull());
+	file.preview.setDevicePixelRatio(style::DevicePixelRatio());
+}
+
+bool ApplyModifications(PreparedList &list) {
+	auto applied = false;
+	const auto apply = [&](PreparedFile &file, QSize strictSize = {}) {
+		const auto image = std::get_if<Image>(&file.information->media);
+		const auto guard = gsl::finally([&] {
+			if (!image || strictSize.isEmpty()) {
+				return;
+			}
+			applied = true;
+			file.path = QString();
+			file.content = QByteArray();
+			image->data = image->data.scaled(
+				strictSize,
+				Qt::IgnoreAspectRatio,
+				Qt::SmoothTransformation);
+		});
+		if (!image || !image->modifications) {
+			return;
+		}
+		applied = true;
+		file.path = QString();
+		file.content = QByteArray();
+		image->data = Editor::ImageModified(
+			std::move(image->data),
+			image->modifications);
+	};
+	for (auto &file : list.files) {
+		apply(file);
+		if (const auto cover = file.videoCover.get()) {
+			const auto video = file.information
+				? std::get_if<Ui::PreparedFileInformation::Video>(
+					&file.information->media)
+				: nullptr;
+			apply(*cover, video ? video->thumbnail.size() : QSize());
+		}
+	}
+	return applied;
+}
+
+} // namespace Storage
+

Telegram/SourceFiles/storage/storage_media_prepare.h

@@ -0,0 +1,66 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#pragma once
+
+#include "core/file_utilities.h"
+
+namespace tr {
+template <typename ...>
+struct phrase;
+} // namespace tr
+
+namespace Ui {
+struct PreparedFileInformation;
+struct PreparedFile;
+struct PreparedList;
+enum class AlbumType;
+} // namespace Ui
+
+namespace Storage {
+
+enum class MimeDataState {
+	None,
+	Files,
+	PhotoFiles,
+	//PremiumFile,
+	Image,
+};
+
+[[nodiscard]] std::optional<Ui::PreparedList> PreparedFileFromFilesDialog(
+	FileDialog::OpenResult &&result,
+	Fn<bool(const Ui::PreparedList&)> checkResult,
+	Fn<void(tr::phrase<>)> errorCallback,
+	int previewWidth,
+	bool premium);
+[[nodiscard]] MimeDataState ComputeMimeDataState(const QMimeData *data);
+[[nodiscard]] bool ValidatePhotoEditorMediaDragData(
+	not_null<const QMimeData*> data);
+[[nodiscard]] bool ValidateEditMediaDragData(
+	not_null<const QMimeData*> data,
+	Ui::AlbumType albumType);
+[[nodiscard]] Ui::PreparedList PrepareMediaList(
+	const QList<QUrl> &files,
+	int previewWidth,
+	bool premium);
+[[nodiscard]] Ui::PreparedList PrepareMediaList(
+	const QStringList &files,
+	int previewWidth,
+	bool premium);
+[[nodiscard]] Ui::PreparedList PrepareMediaFromImage(
+	QImage &&image,
+	QByteArray &&content,
+	int previewWidth);
+void PrepareDetails(Ui::PreparedFile &file, int previewWidth, int sideLimit);
+void UpdateImageDetails(
+	Ui::PreparedFile &file,
+	int previewWidth,
+	int sideLimit);
+
+bool ApplyModifications(Ui::PreparedList &list);
+
+} // namespace Storage

Telegram/SourceFiles/storage/storage_shared_media.cpp

@@ -0,0 +1,215 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#include "storage/storage_shared_media.h"
+
+#include <rpl/map.h>
+
+namespace Storage {
+
+auto SharedMedia::enforceLists(Key key)
+-> std::map<Key, SharedMedia::Lists>::iterator {
+	auto result = _lists.find(key);
+	if (result != _lists.end()) {
+		return result;
+	}
+	result = _lists.emplace(key, Lists {}).first;
+	for (auto index = 0; index != kSharedMediaTypeCount; ++index) {
+		auto &list = result->second[index];
+		auto type = static_cast<SharedMediaType>(index);
+
+		list.sliceUpdated(
+		) | rpl::map([=](const SparseIdsSliceUpdate &update) {
+			return SharedMediaSliceUpdate(
+				key.peerId,
+				key.topicRootId,
+				key.monoforumPeerId,
+				type,
+				update);
+		}) | rpl::start_to_stream(_sliceUpdated, _lifetime);
+	}
+	return result;
+}
+
+void SharedMedia::add(SharedMediaAddNew &&query) {
+	const auto addByIt = [&](const auto i) {
+		for (auto index = 0; index != kSharedMediaTypeCount; ++index) {
+			auto type = static_cast<SharedMediaType>(index);
+			if (query.types.test(type)) {
+				i->second[index].addNew(query.messageId);
+			}
+		}
+	};
+	addByIt(enforceLists({ query.peerId, MsgId(0) }));
+	const auto topicIt = query.topicRootId
+		? _lists.find({ query.peerId, query.topicRootId })
+		: end(_lists);
+	if (topicIt != end(_lists)) {
+		addByIt(topicIt);
+	}
+	const auto monoforumPeerIt = query.monoforumPeerId
+		? _lists.find({ query.peerId, MsgId(), query.monoforumPeerId })
+		: end(_lists);
+	if (monoforumPeerIt != end(_lists)) {
+		addByIt(monoforumPeerIt);
+	}
+}
+
+void SharedMedia::add(SharedMediaAddExisting &&query) {
+	auto peerIt = enforceLists({
+		query.peerId,
+		query.topicRootId,
+		query.monoforumPeerId,
+	});
+	for (auto index = 0; index != kSharedMediaTypeCount; ++index) {
+		auto type = static_cast<SharedMediaType>(index);
+		if (query.types.test(type)) {
+			peerIt->second[index].addExisting(
+				query.messageId,
+				query.noSkipRange);
+		}
+	}
+}
+
+void SharedMedia::add(SharedMediaAddSlice &&query) {
+	Expects(IsValidSharedMediaType(query.type));
+
+	auto peerIt = enforceLists({
+		query.peerId,
+		query.topicRootId,
+		query.monoforumPeerId,
+	});
+	auto index = static_cast<int>(query.type);
+	peerIt->second[index].addSlice(
+		std::move(query.messageIds),
+		query.noSkipRange,
+		query.count);
+}
+
+void SharedMedia::remove(SharedMediaRemoveOne &&query) {
+	auto peerIt = _lists.lower_bound({ query.peerId, MsgId(0) });
+	while (peerIt != end(_lists) && peerIt->first.peerId == query.peerId) {
+		for (auto index = 0; index != kSharedMediaTypeCount; ++index) {
+			auto type = static_cast<SharedMediaType>(index);
+			if (query.types.test(type)) {
+				peerIt->second[index].removeOne(query.messageId);
+			}
+		}
+		++peerIt;
+	}
+	_oneRemoved.fire(std::move(query));
+}
+
+void SharedMedia::remove(SharedMediaRemoveAll &&query) {
+	auto peerIt = _lists.lower_bound({
+		query.peerId,
+		query.topicRootId,
+		query.monoforumPeerId,
+	});
+	while (peerIt != end(_lists)
+		&& peerIt->first.peerId == query.peerId
+		&& (!query.topicRootId
+			|| peerIt->first.topicRootId == query.topicRootId)
+		&& (!query.monoforumPeerId
+			|| peerIt->first.monoforumPeerId == query.monoforumPeerId)) {
+		for (auto index = 0; index != kSharedMediaTypeCount; ++index) {
+			auto type = static_cast<SharedMediaType>(index);
+			if (query.types.test(type)) {
+				peerIt->second[index].removeAll();
+			}
+		}
+		++peerIt;
+	}
+	_allRemoved.fire(std::move(query));
+}
+
+void SharedMedia::invalidate(SharedMediaInvalidateBottom &&query) {
+	auto peerIt = _lists.lower_bound({ query.peerId, MsgId(0) });
+	while (peerIt != end(_lists) && peerIt->first.peerId == query.peerId) {
+		for (auto index = 0; index != kSharedMediaTypeCount; ++index) {
+			peerIt->second[index].invalidateBottom();
+		}
+		++peerIt;
+	}
+	_bottomInvalidated.fire(std::move(query));
+}
+
+void SharedMedia::unload(SharedMediaUnloadThread &&query) {
+	_lists.erase({ query.peerId, query.topicRootId, query.monoforumPeerId });
+}
+
+rpl::producer<SharedMediaResult> SharedMedia::query(SharedMediaQuery &&query) const {
+	Expects(IsValidSharedMediaType(query.key.type));
+
+	auto peerIt = _lists.find({
+		query.key.peerId,
+		query.key.topicRootId,
+		query.key.monoforumPeerId,
+	});
+	if (peerIt != _lists.end()) {
+		auto index = static_cast<int>(query.key.type);
+		return peerIt->second[index].query(SparseIdsListQuery(
+			query.key.messageId,
+			query.limitBefore,
+			query.limitAfter));
+	}
+	return [](auto consumer) {
+		consumer.put_done();
+		return rpl::lifetime();
+	};
+}
+
+SharedMediaResult SharedMedia::snapshot(const SharedMediaQuery &query) const {
+	Expects(IsValidSharedMediaType(query.key.type));
+
+	auto peerIt = _lists.find({
+		query.key.peerId,
+		query.key.topicRootId,
+		query.key.monoforumPeerId,
+	});
+	if (peerIt != _lists.end()) {
+		auto index = static_cast<int>(query.key.type);
+		return peerIt->second[index].snapshot(SparseIdsListQuery(
+			query.key.messageId,
+			query.limitBefore,
+			query.limitAfter));
+	}
+	return {};
+}
+
+bool SharedMedia::empty(const SharedMediaKey &key) const {
+	Expects(IsValidSharedMediaType(key.type));
+
+	auto peerIt = _lists.find({
+		key.peerId,
+		key.topicRootId,
+		key.monoforumPeerId,
+	});
+	if (peerIt != _lists.end()) {
+		auto index = static_cast<int>(key.type);
+		return peerIt->second[index].empty();
+	}
+	return true;
+}
+
+rpl::producer<SharedMediaSliceUpdate> SharedMedia::sliceUpdated() const {
+	return _sliceUpdated.events();
+}
+
+rpl::producer<SharedMediaRemoveOne> SharedMedia::oneRemoved() const {
+	return _oneRemoved.events();
+}
+
+rpl::producer<SharedMediaRemoveAll> SharedMedia::allRemoved() const {
+	return _allRemoved.events();
+}
+
+rpl::producer<SharedMediaInvalidateBottom> SharedMedia::bottomInvalidated() const {
+	return _bottomInvalidated.events();
+}
+
+} // namespace Storage

Telegram/SourceFiles/storage/storage_shared_media.h

@@ -0,0 +1,294 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#pragma once
+
+#include <rpl/event_stream.h>
+#include "storage/storage_facade.h"
+#include "storage/storage_sparse_ids_list.h"
+
+namespace Storage {
+
+// Allow forward declarations.
+enum class SharedMediaType : signed char {
+	Photo,
+	Video,
+	PhotoVideo,
+	MusicFile,
+	File,
+	VoiceFile,
+	Link,
+	ChatPhoto,
+	RoundVoiceFile,
+	GIF,
+	RoundFile,
+	Pinned,
+
+	kCount,
+};
+constexpr auto kSharedMediaTypeCount = static_cast<int>(SharedMediaType::kCount);
+constexpr bool IsValidSharedMediaType(SharedMediaType type) {
+	return (static_cast<int>(type) >= 0)
+		&& (static_cast<int>(type) < kSharedMediaTypeCount);
+}
+
+using SharedMediaTypesMask = base::enum_mask<SharedMediaType>;
+
+struct SharedMediaAddNew {
+	SharedMediaAddNew(
+		PeerId peerId,
+		MsgId topicRootId,
+		PeerId monoforumPeerId,
+		SharedMediaTypesMask types,
+		MsgId messageId)
+	: peerId(peerId)
+	, topicRootId(topicRootId)
+	, monoforumPeerId(monoforumPeerId)
+	, messageId(messageId)
+	, types(types) {
+	}
+
+	PeerId peerId = 0;
+	MsgId topicRootId = 0;
+	PeerId monoforumPeerId = 0;
+	MsgId messageId = 0;
+	SharedMediaTypesMask types;
+
+};
+
+struct SharedMediaAddExisting {
+	SharedMediaAddExisting(
+		PeerId peerId,
+		MsgId topicRootId,
+		PeerId monoforumPeerId,
+		SharedMediaTypesMask types,
+		MsgId messageId,
+		MsgRange noSkipRange)
+	: peerId(peerId)
+	, topicRootId(topicRootId)
+	, monoforumPeerId(monoforumPeerId)
+	, messageId(messageId)
+	, noSkipRange(noSkipRange)
+	, types(types) {
+	}
+
+	PeerId peerId = 0;
+	MsgId topicRootId = 0;
+	PeerId monoforumPeerId = 0;
+	MsgId messageId = 0;
+	MsgRange noSkipRange;
+	SharedMediaTypesMask types;
+
+};
+
+struct SharedMediaAddSlice {
+	SharedMediaAddSlice(
+		PeerId peerId,
+		MsgId topicRootId,
+		PeerId monoforumPeerId,
+		SharedMediaType type,
+		std::vector<MsgId> &&messageIds,
+		MsgRange noSkipRange,
+		std::optional<int> count = std::nullopt)
+	: peerId(peerId)
+	, topicRootId(topicRootId)
+	, monoforumPeerId(monoforumPeerId)
+	, messageIds(std::move(messageIds))
+	, noSkipRange(noSkipRange)
+	, type(type)
+	, count(count) {
+	}
+
+	PeerId peerId = 0;
+	MsgId topicRootId = 0;
+	PeerId monoforumPeerId = 0;
+	std::vector<MsgId> messageIds;
+	MsgRange noSkipRange;
+	SharedMediaType type = SharedMediaType::kCount;
+	std::optional<int> count;
+
+};
+
+struct SharedMediaRemoveOne {
+	SharedMediaRemoveOne(
+		PeerId peerId,
+		SharedMediaTypesMask types,
+		MsgId messageId)
+	: peerId(peerId)
+	, messageId(messageId)
+	, types(types) {
+	}
+
+	PeerId peerId = 0;
+	MsgId messageId = 0;
+	SharedMediaTypesMask types;
+
+};
+
+struct SharedMediaRemoveAll {
+	SharedMediaRemoveAll(
+		PeerId peerId,
+		SharedMediaTypesMask types = SharedMediaTypesMask::All())
+	: peerId(peerId)
+	, types(types) {
+	}
+	SharedMediaRemoveAll(
+		PeerId peerId,
+		MsgId topicRootId,
+		SharedMediaTypesMask types = SharedMediaTypesMask::All())
+	: peerId(peerId)
+	, topicRootId(topicRootId)
+	, types(types) {
+	}
+	SharedMediaRemoveAll(
+		PeerId peerId,
+		PeerId monoforumPeerId,
+		SharedMediaTypesMask types = SharedMediaTypesMask::All())
+	: peerId(peerId)
+	, monoforumPeerId(monoforumPeerId)
+	, types(types) {
+	}
+
+	PeerId peerId = 0;
+	MsgId topicRootId = 0;
+	PeerId monoforumPeerId = 0;
+	SharedMediaTypesMask types;
+
+};
+
+struct SharedMediaInvalidateBottom {
+	SharedMediaInvalidateBottom(PeerId peerId) : peerId(peerId) {
+	}
+
+	PeerId peerId = 0;
+
+};
+
+struct SharedMediaKey {
+	SharedMediaKey(
+		PeerId peerId,
+		MsgId topicRootId,
+		PeerId monoforumPeerId,
+		SharedMediaType type,
+		MsgId messageId)
+	: peerId(peerId)
+	, topicRootId(topicRootId)
+	, monoforumPeerId(monoforumPeerId)
+	, type(type)
+	, messageId(messageId) {
+	}
+
+	friend inline constexpr auto operator<=>(
+		const SharedMediaKey &,
+		const SharedMediaKey &) = default;
+
+	PeerId peerId = 0;
+	MsgId topicRootId = 0;
+	PeerId monoforumPeerId = 0;
+	SharedMediaType type = SharedMediaType::kCount;
+	MsgId messageId = 0;
+
+};
+
+struct SharedMediaQuery {
+	SharedMediaQuery(
+		SharedMediaKey key,
+		int limitBefore,
+		int limitAfter)
+	: key(key)
+	, limitBefore(limitBefore)
+	, limitAfter(limitAfter) {
+	}
+
+	SharedMediaKey key;
+	int limitBefore = 0;
+	int limitAfter = 0;
+
+};
+
+using SharedMediaResult = SparseIdsListResult;
+
+struct SharedMediaSliceUpdate {
+	SharedMediaSliceUpdate(
+		PeerId peerId,
+		MsgId topicRootId,
+		PeerId monoforumPeerId,
+		SharedMediaType type,
+		const SparseIdsSliceUpdate &data)
+	: peerId(peerId)
+	, topicRootId(topicRootId)
+	, monoforumPeerId(monoforumPeerId)
+	, type(type)
+	, data(data) {
+	}
+
+	PeerId peerId = 0;
+	MsgId topicRootId = 0;
+	PeerId monoforumPeerId = 0;
+	SharedMediaType type = SharedMediaType::kCount;
+	SparseIdsSliceUpdate data;
+};
+
+struct SharedMediaUnloadThread {
+	SharedMediaUnloadThread(
+		PeerId peerId,
+		MsgId topicRootId,
+		PeerId monoforumPeerId)
+	: peerId(peerId)
+	, topicRootId(topicRootId)
+	, monoforumPeerId(monoforumPeerId) {
+	}
+
+	PeerId peerId = 0;
+	MsgId topicRootId = 0;
+	PeerId monoforumPeerId = 0;
+};
+
+class SharedMedia {
+public:
+	using Type = SharedMediaType;
+
+	void add(SharedMediaAddNew &&query);
+	void add(SharedMediaAddExisting &&query);
+	void add(SharedMediaAddSlice &&query);
+	void remove(SharedMediaRemoveOne &&query);
+	void remove(SharedMediaRemoveAll &&query);
+	void invalidate(SharedMediaInvalidateBottom &&query);
+	void unload(SharedMediaUnloadThread &&query);
+
+	rpl::producer<SharedMediaResult> query(SharedMediaQuery &&query) const;
+	SharedMediaResult snapshot(const SharedMediaQuery &query) const;
+	bool empty(const SharedMediaKey &key) const;
+	rpl::producer<SharedMediaSliceUpdate> sliceUpdated() const;
+	rpl::producer<SharedMediaRemoveOne> oneRemoved() const;
+	rpl::producer<SharedMediaRemoveAll> allRemoved() const;
+	rpl::producer<SharedMediaInvalidateBottom> bottomInvalidated() const;
+
+private:
+	struct Key {
+		PeerId peerId = 0;
+		MsgId topicRootId = 0;
+		PeerId monoforumPeerId = 0;
+
+		friend inline constexpr auto operator<=>(Key, Key) = default;
+	};
+	using Lists = std::array<SparseIdsList, kSharedMediaTypeCount>;
+
+	std::map<Key, Lists>::iterator enforceLists(Key key);
+
+	std::map<Key, Lists> _lists;
+
+	rpl::event_stream<SharedMediaSliceUpdate> _sliceUpdated;
+	rpl::event_stream<SharedMediaRemoveOne> _oneRemoved;
+	rpl::event_stream<SharedMediaRemoveAll> _allRemoved;
+	rpl::event_stream<SharedMediaInvalidateBottom> _bottomInvalidated;
+
+	rpl::lifetime _lifetime;
+
+};
+
+} // namespace Storage

Telegram/SourceFiles/storage/storage_sparse_ids_list.cpp

@@ -0,0 +1,264 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#include "storage/storage_sparse_ids_list.h"
+
+namespace Storage {
+
+SparseIdsList::Slice::Slice(
+	base::flat_set<MsgId> &&messages,
+	MsgRange range)
+: messages(std::move(messages))
+, range(range) {
+}
+
+template <typename Range>
+void SparseIdsList::Slice::merge(
+		const Range &moreMessages,
+		MsgRange moreNoSkipRange) {
+	Expects(moreNoSkipRange.from <= range.till);
+	Expects(range.from <= moreNoSkipRange.till);
+
+	messages.merge(std::begin(moreMessages), std::end(moreMessages));
+	range = {
+		qMin(range.from, moreNoSkipRange.from),
+		qMax(range.till, moreNoSkipRange.till)
+	};
+}
+
+template <typename Range>
+SparseIdsList::AddResult SparseIdsList::uniteAndAdd(
+		SparseIdsSliceUpdate &update,
+		base::flat_set<Slice>::iterator uniteFrom,
+		base::flat_set<Slice>::iterator uniteTill,
+		const Range &messages,
+		MsgRange noSkipRange) {
+	const auto uniteFromIndex = uniteFrom - _slices.begin();
+	const auto was = int(uniteFrom->messages.size());
+	_slices.modify(uniteFrom, [&](Slice &slice) {
+		slice.merge(messages, noSkipRange);
+	});
+	const auto firstToErase = uniteFrom + 1;
+	if (firstToErase != uniteTill) {
+		for (auto it = firstToErase; it != uniteTill; ++it) {
+			_slices.modify(uniteFrom, [&](Slice &slice) {
+				slice.merge(it->messages, it->range);
+			});
+		}
+		_slices.erase(firstToErase, uniteTill);
+		uniteFrom = _slices.begin() + uniteFromIndex;
+	}
+	update.messages = &uniteFrom->messages;
+	update.range = uniteFrom->range;
+	return { int(uniteFrom->messages.size()) - was };
+}
+
+template <typename Range>
+SparseIdsList::AddResult SparseIdsList::addRangeItemsAndCountNew(
+		SparseIdsSliceUpdate &update,
+		const Range &messages,
+		MsgRange noSkipRange) {
+	Expects(noSkipRange.from <= noSkipRange.till);
+
+	if (noSkipRange.from == noSkipRange.till
+		&& std::begin(messages) == std::end(messages)) {
+		return { 0 };
+	}
+	auto uniteFrom = ranges::lower_bound(
+		_slices,
+		noSkipRange.from,
+		std::less<>(),
+		[](const Slice &slice) { return slice.range.till; });
+	auto uniteTill = ranges::upper_bound(
+		_slices,
+		noSkipRange.till,
+		std::less<>(),
+		[](const Slice &slice) { return slice.range.from; });
+	if (uniteFrom < uniteTill) {
+		return uniteAndAdd(update, uniteFrom, uniteTill, messages, noSkipRange);
+	}
+
+	auto sliceMessages = base::flat_set<MsgId> {
+		std::begin(messages),
+		std::end(messages) };
+	auto slice = _slices.emplace(
+		std::move(sliceMessages),
+		noSkipRange
+	).first;
+	update.messages = &slice->messages;
+	update.range = slice->range;
+	return { int(slice->messages.size()) };
+}
+
+template <typename Range>
+void SparseIdsList::addRange(
+		const Range &messages,
+		MsgRange noSkipRange,
+		std::optional<int> count,
+		bool incrementCount) {
+	Expects(!count || !incrementCount);
+
+	auto update = SparseIdsSliceUpdate();
+	const auto result = addRangeItemsAndCountNew(
+		update,
+		messages,
+		noSkipRange);
+	if (count) {
+		_count = count;
+	} else if (incrementCount && _count && result.added > 0) {
+		*_count += result.added;
+	}
+	if (_slices.size() == 1) {
+		if (_count && _slices.front().messages.size() >= *_count) {
+			_slices.modify(_slices.begin(), [&](Slice &slice) {
+				slice.range = { 0, ServerMaxMsgId };
+			});
+		}
+		if (_slices.front().range == MsgRange{ 0, ServerMaxMsgId }) {
+			_count = _slices.front().messages.size();
+		}
+	}
+	if (_count && update.messages) {
+		accumulate_max(*_count, int(update.messages->size()));
+	}
+	update.count = _count;
+	_sliceUpdated.fire(std::move(update));
+}
+
+void SparseIdsList::addNew(MsgId messageId) {
+	auto range = { messageId };
+	addRange(range, { messageId, ServerMaxMsgId }, std::nullopt, true);
+}
+
+void SparseIdsList::addExisting(
+		MsgId messageId,
+		MsgRange noSkipRange) {
+	auto range = { messageId };
+	addRange(range, noSkipRange, std::nullopt);
+}
+
+void SparseIdsList::addSlice(
+		std::vector<MsgId> &&messageIds,
+		MsgRange noSkipRange,
+		std::optional<int> count) {
+	addRange(messageIds, noSkipRange, count);
+}
+
+void SparseIdsList::removeOne(MsgId messageId) {
+	auto slice = ranges::lower_bound(
+		_slices,
+		messageId,
+		std::less<>(),
+		[](const Slice &slice) { return slice.range.till; });
+	if (slice != _slices.end() && slice->range.from <= messageId) {
+		_slices.modify(slice, [messageId](Slice &slice) {
+			return slice.messages.remove(messageId);
+		});
+	}
+	if (_count) {
+		--*_count;
+	}
+}
+
+void SparseIdsList::removeAll() {
+	_slices.clear();
+	_slices.emplace(base::flat_set<MsgId>{}, MsgRange { 0, ServerMaxMsgId });
+	_count = 0;
+}
+
+void SparseIdsList::invalidateBottom() {
+	if (!_slices.empty()) {
+		const auto &last = _slices.back();
+		if (last.range.till == ServerMaxMsgId) {
+			_slices.modify(_slices.end() - 1, [](Slice &slice) {
+				slice.range.till = slice.messages.empty()
+					? slice.range.from
+					: slice.messages.back();
+			});
+		}
+	}
+	_count = std::nullopt;
+}
+
+rpl::producer<SparseIdsListResult> SparseIdsList::query(
+		SparseIdsListQuery &&query) const {
+	return [this, query = std::move(query)](auto consumer) {
+		auto now = snapshot(query);
+		if (!now.messageIds.empty() || now.count) {
+			consumer.put_next(std::move(now));
+		}
+		consumer.put_done();
+		return rpl::lifetime();
+	};
+}
+
+SparseIdsListResult SparseIdsList::snapshot(
+		const SparseIdsListQuery &query) const {
+	auto slice = query.aroundId
+		? ranges::lower_bound(
+			_slices,
+			query.aroundId,
+			std::less<>(),
+			[](const Slice &slice) { return slice.range.till; })
+		: _slices.end();
+	if (slice != _slices.end()
+		&& slice->range.from <= query.aroundId) {
+		return queryFromSlice(query, *slice);
+	} else if (_count) {
+		auto result = SparseIdsListResult{};
+		result.count = _count;
+		return result;
+	}
+	return {};
+}
+
+bool SparseIdsList::empty() const {
+	for (const auto &slice : _slices) {
+		if (!slice.messages.empty()) {
+			return false;
+		}
+	}
+	return true;
+}
+
+rpl::producer<SparseIdsSliceUpdate> SparseIdsList::sliceUpdated() const {
+	return _sliceUpdated.events();
+}
+
+SparseIdsListResult SparseIdsList::queryFromSlice(
+		const SparseIdsListQuery &query,
+		const Slice &slice) const {
+	auto result = SparseIdsListResult {};
+	auto position = ranges::lower_bound(slice.messages, query.aroundId);
+	auto haveBefore = int(position - slice.messages.begin());
+	auto haveEqualOrAfter = int(slice.messages.end() - position);
+	auto before = qMin(haveBefore, query.limitBefore);
+	auto equalOrAfter = qMin(haveEqualOrAfter, query.limitAfter + 1);
+	auto ids = std::vector<MsgId>(position - before, position + equalOrAfter);
+	result.messageIds.merge(ids.begin(), ids.end());
+	if (slice.range.from == 0) {
+		result.skippedBefore = haveBefore - before;
+	}
+	if (slice.range.till == ServerMaxMsgId) {
+		result.skippedAfter = haveEqualOrAfter - equalOrAfter;
+	}
+	if (_count) {
+		result.count = _count;
+		if (!result.skippedBefore && result.skippedAfter) {
+			result.skippedBefore = *result.count
+				- *result.skippedAfter
+				- int(result.messageIds.size());
+		} else if (!result.skippedAfter && result.skippedBefore) {
+			result.skippedAfter = *result.count
+				- *result.skippedBefore
+				- int(result.messageIds.size());
+		}
+	}
+	return result;
+}
+
+} // namespace Storage

Telegram/SourceFiles/storage/storage_sparse_ids_list.h

@@ -0,0 +1,106 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#pragma once
+
+namespace Storage {
+
+struct SparseIdsListQuery {
+	SparseIdsListQuery(
+		MsgId aroundId,
+		int limitBefore,
+		int limitAfter)
+	: aroundId(aroundId)
+	, limitBefore(limitBefore)
+	, limitAfter(limitAfter) {
+	}
+
+	MsgId aroundId = 0;
+	int limitBefore = 0;
+	int limitAfter = 0;
+
+};
+
+struct SparseIdsListResult {
+	std::optional<int> count;
+	std::optional<int> skippedBefore;
+	std::optional<int> skippedAfter;
+	base::flat_set<MsgId> messageIds;
+};
+
+struct SparseIdsSliceUpdate {
+	const base::flat_set<MsgId> *messages = nullptr;
+	MsgRange range;
+	std::optional<int> count;
+};
+
+class SparseIdsList {
+public:
+	void addNew(MsgId messageId);
+	void addExisting(MsgId messageId, MsgRange noSkipRange);
+	void addSlice(
+		std::vector<MsgId> &&messageIds,
+		MsgRange noSkipRange,
+		std::optional<int> count);
+	void removeOne(MsgId messageId);
+	void removeAll();
+	void invalidateBottom();
+	rpl::producer<SparseIdsListResult> query(SparseIdsListQuery &&query) const;
+	rpl::producer<SparseIdsSliceUpdate> sliceUpdated() const;
+	SparseIdsListResult snapshot(const SparseIdsListQuery &query) const;
+	bool empty() const;
+
+private:
+	struct Slice {
+		Slice(base::flat_set<MsgId> &&messages, MsgRange range);
+
+		template <typename Range>
+		void merge(const Range &moreMessages, MsgRange moreNoSkipRange);
+
+		base::flat_set<MsgId> messages;
+		MsgRange range;
+
+		inline bool operator<(const Slice &other) const {
+			return range.from < other.range.from;
+		}
+
+	};
+
+	struct AddResult {
+		int added = 0;
+	};
+	template <typename Range>
+	AddResult uniteAndAdd(
+		SparseIdsSliceUpdate &update,
+		base::flat_set<Slice>::iterator uniteFrom,
+		base::flat_set<Slice>::iterator uniteTill,
+		const Range &messages,
+		MsgRange noSkipRange);
+	template <typename Range>
+	AddResult addRangeItemsAndCountNew(
+		SparseIdsSliceUpdate &update,
+		const Range &messages,
+		MsgRange noSkipRange);
+	template <typename Range>
+	void addRange(
+		const Range &messages,
+		MsgRange noSkipRange,
+		std::optional<int> count,
+		bool incrementCount = false);
+
+	SparseIdsListResult queryFromSlice(
+		const SparseIdsListQuery &query,
+		const Slice &slice) const;
+
+	std::optional<int> _count;
+	base::flat_set<Slice> _slices;
+
+	rpl::event_stream<SparseIdsSliceUpdate> _sliceUpdated;
+
+};
+
+} // namespace Storage

Telegram/SourceFiles/storage/storage_user_photos.cpp

@@ -0,0 +1,221 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#include "storage/storage_user_photos.h"
+
+namespace Storage {
+
+void UserPhotos::List::setBack(PhotoId photoId) {
+	if (_backPhotoId != photoId) {
+		detachBack();
+		_backPhotoId = photoId;
+		attachBack();
+		sendUpdate();
+	}
+}
+
+void UserPhotos::List::detachBack() {
+	if (_backPhotoId) {
+		removeOne(_backPhotoId);
+	}
+}
+
+void UserPhotos::List::attachBack() {
+	if (_backPhotoId) {
+		_photoIds.push_front(_backPhotoId);
+		if (_count) {
+			++*_count;
+		}
+	}
+}
+
+void UserPhotos::List::addNew(PhotoId photoId) {
+	if (!base::contains(_photoIds, photoId)) {
+		detachBack();
+		_photoIds.push_back(photoId);
+		if (_count) {
+			++*_count;
+		}
+		attachBack();
+		sendUpdate();
+	}
+}
+
+void UserPhotos::List::addSlice(
+		std::vector<PhotoId> &&photoIds,
+		int count) {
+	detachBack();
+	for (auto photoId : photoIds) {
+		if (!base::contains(_photoIds, photoId)) {
+			_photoIds.push_front(photoId);
+		}
+	}
+
+	_count = count;
+	if ((_count && *_count < _photoIds.size()) || photoIds.empty()) {
+		_count = _photoIds.size();
+	}
+	attachBack();
+	sendUpdate();
+}
+
+void UserPhotos::List::removeOne(PhotoId photoId) {
+	auto position = ranges::find(_photoIds, photoId);
+	if (position == _photoIds.end()) {
+		_count = std::nullopt;
+	} else {
+		if (_count) {
+			--*_count;
+		}
+		_photoIds.erase(position);
+	}
+	sendUpdate();
+}
+
+void UserPhotos::List::removeAfter(PhotoId photoId) {
+	auto position = ranges::find(_photoIds, photoId);
+	if (position == _photoIds.end()) {
+		_count = std::nullopt;
+		_photoIds.clear();
+	} else {
+		if (_count) {
+			*_count -= (_photoIds.end() - position);
+		}
+		_photoIds.erase(position, _photoIds.end());
+	}
+	sendUpdate();
+}
+
+void UserPhotos::List::replace(PhotoId oldPhotoId, PhotoId newPhotoId) {
+	auto position = ranges::find(_photoIds, oldPhotoId);
+	if (position != _photoIds.end()) {
+		*position = newPhotoId;
+	}
+	if (_backPhotoId == oldPhotoId) {
+		_backPhotoId = newPhotoId;
+	}
+	sendUpdate();
+}
+
+void UserPhotos::List::sendUpdate() {
+	auto update = SliceUpdate();
+	update.photoIds = &_photoIds;
+	update.count = _count;
+	_sliceUpdated.fire(std::move(update));
+}
+
+rpl::producer<UserPhotosResult> UserPhotos::List::query(
+		UserPhotosQuery &&query) const {
+	return [this, query = std::move(query)](auto consumer) {
+		auto result = UserPhotosResult();
+		result.count = _count;
+
+		auto position = ranges::find(_photoIds, query.key.photoId);
+		if (position != _photoIds.end()) {
+			auto haveBefore = int(position - _photoIds.begin());
+			auto haveEqualOrAfter = int(_photoIds.end() - position);
+			auto before = qMin(haveBefore, query.limitBefore);
+			auto equalOrAfter = qMin(haveEqualOrAfter, query.limitAfter + 1);
+			result.photoIds = std::deque<PhotoId>(
+				position - before,
+				position + equalOrAfter);
+
+			auto skippedInIds = (haveBefore - before);
+			result.skippedBefore = _count
+				| func::add(-int(_photoIds.size()) + skippedInIds);
+			result.skippedBefore = haveBefore - before;
+			result.skippedAfter = (haveEqualOrAfter - equalOrAfter);
+			consumer.put_next(std::move(result));
+		} else if (query.key.back && _backPhotoId) {
+			result.photoIds.push_front(_backPhotoId);
+			result.count = 1;
+			consumer.put_next(std::move(result));
+		} else if (_count) {
+			consumer.put_next(std::move(result));
+		}
+		consumer.put_done();
+		return rpl::lifetime();
+	};
+}
+
+auto UserPhotos::List::sliceUpdated() const -> rpl::producer<SliceUpdate> {
+	return _sliceUpdated.events();
+}
+
+rpl::producer<UserPhotosSliceUpdate> UserPhotos::sliceUpdated() const {
+	return _sliceUpdated.events();
+}
+
+std::map<UserId, UserPhotos::List>::iterator UserPhotos::enforceLists(
+		UserId user) {
+	auto result = _lists.find(user);
+	if (result != _lists.end()) {
+		return result;
+	}
+	result = _lists.emplace(user, List {}).first;
+	result->second.sliceUpdated(
+	) | rpl::on_next([this, user](
+			const SliceUpdate &update) {
+		_sliceUpdated.fire(UserPhotosSliceUpdate(
+			user,
+			update.photoIds,
+			update.count));
+	}, _lifetime);
+	return result;
+}
+
+void UserPhotos::add(UserPhotosSetBack &&query) {
+	auto userIt = enforceLists(query.userId);
+	userIt->second.setBack(query.photoId);
+}
+
+void UserPhotos::add(UserPhotosAddNew &&query) {
+	auto userIt = enforceLists(query.userId);
+	userIt->second.addNew(query.photoId);
+}
+
+void UserPhotos::add(UserPhotosAddSlice &&query) {
+	auto userIt = enforceLists(query.userId);
+	userIt->second.addSlice(
+		std::move(query.photoIds),
+		query.count);
+}
+
+void UserPhotos::remove(UserPhotosRemoveOne &&query) {
+	auto userIt = _lists.find(query.userId);
+	if (userIt != _lists.end()) {
+		userIt->second.removeOne(query.photoId);
+	}
+}
+
+void UserPhotos::remove(UserPhotosRemoveAfter &&query) {
+	auto userIt = _lists.find(query.userId);
+	if (userIt != _lists.end()) {
+		userIt->second.removeAfter(query.photoId);
+	}
+}
+
+void UserPhotos::replace(UserPhotosReplace &&query) {
+	auto userIt = _lists.find(query.userId);
+	if (userIt != _lists.end()) {
+		userIt->second.replace(query.oldPhotoId, query.newPhotoId);
+	}
+}
+
+rpl::producer<UserPhotosResult> UserPhotos::query(
+		UserPhotosQuery &&query) const {
+	auto userIt = _lists.find(query.key.userId);
+	if (userIt != _lists.end()) {
+		return userIt->second.query(std::move(query));
+	}
+	return [](auto consumer) {
+		consumer.put_done();
+		return rpl::lifetime();
+	};
+}
+
+} // namespace Storage

Telegram/SourceFiles/storage/storage_user_photos.h

@@ -0,0 +1,212 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#pragma once
+
+#include <rpl/event_stream.h>
+#include "storage/storage_facade.h"
+
+namespace Storage {
+
+struct UserPhotosSetBack {
+	UserPhotosSetBack(UserId userId, PhotoId photoId)
+		: userId(userId), photoId(photoId) {
+	}
+
+	UserId userId = 0;
+	PhotoId photoId = 0;
+
+};
+
+struct UserPhotosAddNew {
+	UserPhotosAddNew(UserId userId, PhotoId photoId)
+		: userId(userId), photoId(photoId) {
+	}
+
+	UserId userId = 0;
+	PhotoId photoId = 0;
+
+};
+
+struct UserPhotosAddSlice {
+	UserPhotosAddSlice(
+		UserId userId,
+		std::vector<PhotoId> &&photoIds,
+		int count)
+		: userId(userId)
+		, photoIds(std::move(photoIds))
+		, count(count) {
+	}
+
+	UserId userId = 0;
+	std::vector<PhotoId> photoIds;
+	int count = 0;
+
+};
+
+struct UserPhotosRemoveOne {
+	UserPhotosRemoveOne(
+		UserId userId,
+		PhotoId photoId)
+	: userId(userId)
+	, photoId(photoId) {
+	}
+
+	UserId userId = 0;
+	PhotoId photoId = 0;
+
+};
+
+struct UserPhotosRemoveAfter {
+	UserPhotosRemoveAfter(
+		UserId userId,
+		PhotoId photoId)
+		: userId(userId)
+		, photoId(photoId) {
+	}
+
+	UserId userId = 0;
+	PhotoId photoId = 0;
+
+};
+
+struct UserPhotosReplace {
+	UserPhotosReplace(
+		UserId userId,
+		PhotoId oldPhotoId,
+		PhotoId newPhotoId)
+		: userId(userId)
+		, oldPhotoId(oldPhotoId)
+		, newPhotoId(newPhotoId) {
+	}
+
+	UserId userId = 0;
+	PhotoId oldPhotoId = 0;
+	PhotoId newPhotoId = 0;
+
+};
+
+struct UserPhotosKey {
+	UserPhotosKey(
+		UserId userId,
+		PhotoId photoId)
+		: userId(userId)
+		, photoId(photoId) {
+	}
+	UserPhotosKey(UserId userId, bool back) : userId(userId), back(back) {
+	}
+
+	bool operator==(const UserPhotosKey &other) const {
+		return (userId == other.userId)
+			&& (photoId == other.photoId)
+			&& (back == other.back);
+	}
+	bool operator!=(const UserPhotosKey &other) const {
+		return !(*this == other);
+	}
+
+	UserId userId = 0;
+	PhotoId photoId = 0;
+	bool back = false;
+
+};
+
+struct UserPhotosQuery {
+	UserPhotosQuery(
+		UserPhotosKey key,
+		int limitBefore,
+		int limitAfter)
+		: key(key)
+		, limitBefore(limitBefore)
+		, limitAfter(limitAfter) {
+	}
+
+	UserPhotosKey key;
+	int limitBefore = 0;
+	int limitAfter = 0;
+
+};
+
+struct UserPhotosResult {
+	std::optional<int> count;
+	std::optional<int> skippedBefore;
+	int skippedAfter = 0;
+	std::deque<PhotoId> photoIds;
+};
+
+struct UserPhotosSliceUpdate {
+	UserPhotosSliceUpdate(
+		UserId userId,
+		const std::deque<PhotoId> *photoIds,
+		std::optional<int> count)
+		: userId(userId)
+		, photoIds(photoIds)
+		, count(count) {
+	}
+
+	UserId userId = 0;
+	const std::deque<PhotoId> *photoIds = nullptr;
+	std::optional<int> count;
+};
+
+class UserPhotos {
+public:
+	void add(UserPhotosSetBack &&query);
+	void add(UserPhotosAddNew &&query);
+	void add(UserPhotosAddSlice &&query);
+	void remove(UserPhotosRemoveOne &&query);
+	void remove(UserPhotosRemoveAfter &&query);
+	void replace(UserPhotosReplace &&query);
+
+	rpl::producer<UserPhotosResult> query(UserPhotosQuery &&query) const;
+	rpl::producer<UserPhotosSliceUpdate> sliceUpdated() const;
+
+private:
+	class List {
+	public:
+		void setBack(PhotoId photoId);
+		void addNew(PhotoId photoId);
+		void addSlice(
+			std::vector<PhotoId> &&photoIds,
+			int count);
+		void removeOne(PhotoId photoId);
+		void removeAfter(PhotoId photoId);
+		void replace(PhotoId oldPhotoId, PhotoId newPhotoId);
+		rpl::producer<UserPhotosResult> query(UserPhotosQuery &&query) const;
+
+		struct SliceUpdate {
+			const std::deque<PhotoId> *photoIds = nullptr;
+			std::optional<int> count;
+		};
+		rpl::producer<SliceUpdate> sliceUpdated() const;
+
+	private:
+		void sendUpdate();
+		void detachBack();
+		void attachBack();
+
+		std::optional<int> _count;
+		std::deque<PhotoId> _photoIds;
+
+		PhotoId _backPhotoId = PhotoId(0);
+
+		rpl::event_stream<SliceUpdate> _sliceUpdated;
+
+	};
+	using SliceUpdate = List::SliceUpdate;
+
+	std::map<UserId, List>::iterator enforceLists(UserId user);
+
+	std::map<UserId, List> _lists;
+
+	rpl::event_stream<UserPhotosSliceUpdate> _sliceUpdated;
+
+	rpl::lifetime _lifetime;
+
+};
+
+} // namespace Storage

Telegram/SourceFiles/storage/streamed_file_downloader.cpp

@@ -0,0 +1,172 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#include "storage/streamed_file_downloader.h"
+
+#include "media/streaming/media_streaming_loader.h"
+#include "media/streaming/media_streaming_reader.h"
+
+namespace Storage {
+namespace {
+
+using namespace Media::Streaming;
+
+constexpr auto kPartSize = Loader::kPartSize;
+constexpr auto kRequestPartsCount = 32;
+
+} // namespace
+
+StreamedFileDownloader::StreamedFileDownloader(
+	not_null<Main::Session*> session,
+
+	uint64 objectId,
+	MTP::DcId dcId,
+	Data::FileOrigin origin,
+	Cache::Key cacheKey,
+	MediaKey fileLocationKey,
+	std::shared_ptr<Reader> reader,
+
+	// For FileLoader
+	const QString &toFile,
+	int64 size,
+	LocationType locationType,
+	LoadToCacheSetting toCache,
+	LoadFromCloudSetting fromCloud,
+	bool autoLoading,
+	uint8 cacheTag)
+: FileLoader(
+	session,
+	toFile,
+	size,
+	size,
+	locationType,
+	toCache,
+	fromCloud,
+	autoLoading,
+	cacheTag)
+, _objectId(objectId)
+, _origin(origin)
+, _cacheKey(cacheKey)
+, _fileLocationKey(fileLocationKey)
+, _reader(std::move(reader))
+, _partsCount((size + kPartSize - 1) / kPartSize) {
+	_partIsSaved.resize(_partsCount, false);
+
+	_reader->partsForDownloader(
+	) | rpl::on_next([=](const LoadedPart &part) {
+		if (part.offset == LoadedPart::kFailedOffset) {
+			cancel(FailureReason::OtherFailure);
+		} else {
+			savePart(std::move(part));
+		}
+	}, _lifetime);
+}
+
+StreamedFileDownloader::~StreamedFileDownloader() {
+	if (!_finished) {
+		cancel();
+	} else {
+		_reader->cancelForDownloader(this);
+	}
+}
+
+uint64 StreamedFileDownloader::objId() const {
+	return _objectId;
+}
+
+Data::FileOrigin StreamedFileDownloader::fileOrigin() const {
+	return _origin;
+}
+
+void StreamedFileDownloader::requestParts() {
+	while (!_finished
+		&& _nextPartIndex < _partsCount
+		&& _partsRequested < kRequestPartsCount) {
+		requestPart();
+	}
+	_reader->continueDownloaderFromMainThread();
+}
+
+void StreamedFileDownloader::requestPart() {
+	Expects(!_finished);
+
+	const auto index = std::find(
+		begin(_partIsSaved) + _nextPartIndex,
+		end(_partIsSaved),
+		false
+	) - begin(_partIsSaved);
+	if (index == _partsCount) {
+		_nextPartIndex = _partsCount;
+		return;
+	}
+	_nextPartIndex = index + 1;
+	_reader->loadForDownloader(this, index * kPartSize);
+	++_partsRequested;
+}
+
+QByteArray StreamedFileDownloader::readLoadedPart(int64 offset) {
+	Expects(offset >= 0 && offset < _fullSize);
+	Expects(!(offset % kPartSize));
+
+	const auto index = (offset / kPartSize);
+	return _partIsSaved[index]
+		? readLoadedPartBack(offset, kPartSize)
+		: QByteArray();
+}
+
+Storage::Cache::Key StreamedFileDownloader::cacheKey() const {
+	return _cacheKey;
+}
+
+std::optional<MediaKey> StreamedFileDownloader::fileLocationKey() const {
+	return _fileLocationKey;
+}
+
+void StreamedFileDownloader::cancelHook() {
+	_partsRequested = 0;
+	_nextPartIndex = 0;
+
+	_reader->cancelForDownloader(this);
+}
+
+void StreamedFileDownloader::startLoading() {
+	requestParts();
+}
+
+void StreamedFileDownloader::savePart(const LoadedPart &part) {
+	Expects(part.offset >= 0 && part.offset < _reader->size());
+	Expects(part.offset % kPartSize == 0);
+
+	if (_finished || _cancelled) {
+		return;
+	}
+
+	const auto offset = part.offset;
+	const auto index = offset / kPartSize;
+	Assert(index >= 0 && index < _partsCount);
+	if (_partIsSaved[index]) {
+		return;
+	}
+	_partIsSaved[index] = true;
+	++_partsSaved;
+
+	if (index < _nextPartIndex) {
+		--_partsRequested;
+	}
+	if (!writeResultPart(offset, bytes::make_span(part.bytes))) {
+		return;
+	}
+	_reader->doneForDownloader(offset);
+	if (_partsSaved == _partsCount) {
+		finalizeResult();
+	} else {
+		requestParts();
+		notifyAboutProgress();
+	}
+}
+
+} // namespace Storage

Telegram/SourceFiles/storage/streamed_file_downloader.h

@@ -0,0 +1,76 @@
+/*
+This file is part of Telegram Desktop,
+the official desktop application for the Telegram messaging service.
+
+For license and copyright information please follow this link:
+https://github.com/telegramdesktop/tdesktop/blob/master/LEGAL
+*/
+#pragma once
+
+#include "storage/file_download.h"
+#include "storage/cache/storage_cache_types.h"
+#include "data/data_file_origin.h"
+
+namespace Media {
+namespace Streaming {
+class Reader;
+struct LoadedPart;
+} // namespace Streaming
+} // namespace Media
+
+namespace Storage {
+
+class StreamedFileDownloader final : public FileLoader {
+public:
+	StreamedFileDownloader(
+		not_null<Main::Session*> session,
+
+		uint64 objectId,
+		MTP::DcId dcId,
+		Data::FileOrigin origin,
+		Cache::Key cacheKey,
+		MediaKey fileLocationKey,
+		std::shared_ptr<Media::Streaming::Reader> reader,
+
+		// For FileLoader
+		const QString &toFile,
+		int64 size,
+		LocationType locationType,
+		LoadToCacheSetting toCache,
+		LoadFromCloudSetting fromCloud,
+		bool autoLoading,
+		uint8 cacheTag);
+	~StreamedFileDownloader();
+
+	uint64 objId() const override;
+	Data::FileOrigin fileOrigin() const override;
+
+	QByteArray readLoadedPart(int64 offset);
+
+private:
+	void startLoading() override;
+	Cache::Key cacheKey() const override;
+	std::optional<MediaKey> fileLocationKey() const override;
+	void cancelHook() override;
+	void requestParts();
+	void requestPart();
+
+	void savePart(const Media::Streaming::LoadedPart &part);
+
+	uint64 _objectId = 0;
+	Data::FileOrigin _origin;
+	Cache::Key _cacheKey;
+	MediaKey _fileLocationKey;
+	std::shared_ptr<Media::Streaming::Reader> _reader;
+
+	std::vector<bool> _partIsSaved; // vector<bool> :D
+	mutable int _nextPartIndex = 0;
+	int _partsCount = 0;
+	int _partsRequested = 0;
+	int _partsSaved = 0;
+
+	rpl::lifetime _lifetime;
+
+};
+
+} // namespace Storage