From 93cf57913c415952529eb0692f82b73f8a8cb575 Mon Sep 17 00:00:00 2001 From: Gabriel A Date: Wed, 12 Jul 2023 16:17:58 -0300 Subject: [PATCH] Remove address space mirror and tweak address space layout when host has small adress space --- src/Ryujinx.Cpu/AddressSpace.cs | 36 +- src/Ryujinx.Cpu/AppleHv/HvMemoryManager.cs | 836 +---------------- .../Jit/MemoryManagerHostNoMirror.cs | 111 +++ src/Ryujinx.Cpu/MemoryEhMeilleure.cs | 2 +- src/Ryujinx.Cpu/MemoryManagerSoftware.cs | 858 ++++++++++++++++++ src/Ryujinx.Cpu/Nce/MemoryManagerNative.cs | 745 +-------------- .../HOS/ArmProcessContextFactory.cs | 41 +- .../HOS/Kernel/Memory/KPageTableBase.cs | 12 +- 8 files changed, 1046 insertions(+), 1595 deletions(-) create mode 100644 src/Ryujinx.Cpu/Jit/MemoryManagerHostNoMirror.cs create mode 100644 src/Ryujinx.Cpu/MemoryManagerSoftware.cs diff --git a/src/Ryujinx.Cpu/AddressSpace.cs b/src/Ryujinx.Cpu/AddressSpace.cs index beea14bee..4e1820f01 100644 --- a/src/Ryujinx.Cpu/AddressSpace.cs +++ b/src/Ryujinx.Cpu/AddressSpace.cs @@ -181,26 +181,44 @@ namespace Ryujinx.Cpu const MemoryAllocationFlags AsFlags = MemoryAllocationFlags.Reserve | MemoryAllocationFlags.ViewCompatible; + MemoryBlock baseMemory = null; + MemoryBlock mirrorMemory = null; + + try + { + baseMemory = new MemoryBlock(asSize, asFlags); + mirrorMemory = new MemoryBlock(asSize, asFlags); + addressSpace = new AddressSpace(backingMemory, baseMemory, mirrorMemory, asSize, supports4KBPages); + } + catch (SystemException) + { + baseMemory?.Dispose(); + mirrorMemory?.Dispose(); + } + + return addressSpace != null; + } + + public static bool TryCreateWithoutMirror(ulong asSize, out MemoryBlock addressSpace) + { + addressSpace = null; + + MemoryAllocationFlags asFlags = MemoryAllocationFlags.Reserve | MemoryAllocationFlags.ViewCompatible; + ulong minAddressSpaceSize = Math.Min(asSize, 1UL << 36); // Attempt to create the address space with expected size or try to reduce it until it succeed. - for (ulong addressSpaceSize = asSize; addressSpaceSize >= minAddressSpaceSize; addressSpaceSize >>= 1) + for (ulong addressSpaceSize = asSize; addressSpaceSize >= minAddressSpaceSize; addressSpaceSize -= 0x100000000UL) { - MemoryBlock baseMemory = null; - MemoryBlock mirrorMemory = null; - try { - baseMemory = new MemoryBlock(addressSpaceSize, AsFlags); - mirrorMemory = new MemoryBlock(addressSpaceSize, AsFlags); - addressSpace = new AddressSpace(backingMemory, baseMemory, mirrorMemory, addressSpaceSize, supports4KBPages); + MemoryBlock baseMemory = new MemoryBlock(addressSpaceSize, AsFlags); + addressSpace = baseMemory; break; } catch (SystemException) { - baseMemory?.Dispose(); - mirrorMemory?.Dispose(); } } diff --git a/src/Ryujinx.Cpu/AppleHv/HvMemoryManager.cs b/src/Ryujinx.Cpu/AppleHv/HvMemoryManager.cs index 41e5ec7de..03f4b58fb 100644 --- a/src/Ryujinx.Cpu/AppleHv/HvMemoryManager.cs +++ b/src/Ryujinx.Cpu/AppleHv/HvMemoryManager.cs @@ -1,10 +1,7 @@ using ARMeilleure.Memory; using Ryujinx.Memory; -using Ryujinx.Memory.Range; using Ryujinx.Memory.Tracking; using System; -using System.Collections.Generic; -using System.Linq; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Versioning; @@ -16,27 +13,8 @@ namespace Ryujinx.Cpu.AppleHv /// Represents a CPU memory manager which maps guest virtual memory directly onto the Hypervisor page table. /// [SupportedOSPlatform("macos")] - public class HvMemoryManager : MemoryManagerBase, ICpuMemoryManager, IVirtualMemoryManagerTracked, IWritableBlock + public class HvMemoryManager : MemoryManagerSoftware, ICpuMemoryManager, IVirtualMemoryManagerTracked, IWritableBlock { - public const int PageBits = 12; - public const int PageSize = 1 << PageBits; - public const int PageMask = PageSize - 1; - - public const int PageToPteShift = 5; // 32 pages (2 bits each) in one ulong page table entry. - public const ulong BlockMappedMask = 0x5555555555555555; // First bit of each table entry set. - - private enum HostMappedPtBits : ulong - { - Unmapped = 0, - Mapped, - WriteTracked, - ReadWriteTracked, - - MappedReplicated = 0x5555555555555555, - WriteTrackedReplicated = 0xaaaaaaaaaaaaaaaa, - ReadWriteTrackedReplicated = ulong.MaxValue, - } - private readonly InvalidAccessHandler _invalidAccessHandler; private readonly ulong _addressSpaceSize; @@ -45,21 +23,12 @@ namespace Ryujinx.Cpu.AppleHv internal HvAddressSpace AddressSpace => _addressSpace; - private readonly MemoryBlock _backingMemory; - private readonly PageTable _pageTable; - - private readonly ulong[] _pageBitmap; - public bool Supports4KBPages => true; - public int AddressSpaceBits { get; } - public IntPtr PageTablePointer => IntPtr.Zero; public MemoryManagerType Type => MemoryManagerType.SoftwarePageTable; - public MemoryTracking Tracking { get; } - public event Action UnmapEvent; /// @@ -68,66 +37,16 @@ namespace Ryujinx.Cpu.AppleHv /// Physical backing memory where virtual memory will be mapped to /// Size of the address space /// Optional function to handle invalid memory accesses - public HvMemoryManager(MemoryBlock backingMemory, ulong addressSpaceSize, InvalidAccessHandler invalidAccessHandler = null) + public HvMemoryManager(MemoryBlock backingMemory, ulong addressSpaceSize, InvalidAccessHandler invalidAccessHandler = null) : base(backingMemory, addressSpaceSize, invalidAccessHandler) { - _backingMemory = backingMemory; - _pageTable = new PageTable(); _invalidAccessHandler = invalidAccessHandler; _addressSpaceSize = addressSpaceSize; - ulong asSize = PageSize; - int asBits = PageBits; + _addressSpace = new HvAddressSpace(backingMemory, addressSpaceSize); - while (asSize < addressSpaceSize) - { - asSize <<= 1; - asBits++; - } - - _addressSpace = new HvAddressSpace(backingMemory, asSize); - - AddressSpaceBits = asBits; - - _pageBitmap = new ulong[1 << (AddressSpaceBits - (PageBits + PageToPteShift))]; Tracking = new MemoryTracking(this, PageSize, invalidAccessHandler); } - /// - /// Checks if the virtual address is part of the addressable space. - /// - /// Virtual address - /// True if the virtual address is part of the addressable space - private bool ValidateAddress(ulong va) - { - return va < _addressSpaceSize; - } - - /// - /// Checks if the combination of virtual address and size is part of the addressable space. - /// - /// Virtual address of the range - /// Size of the range in bytes - /// True if the combination of virtual address and size is part of the addressable space - private bool ValidateAddressAndSize(ulong va, ulong size) - { - ulong endVa = va + size; - return endVa >= va && endVa >= size && endVa <= _addressSpaceSize; - } - - /// - /// Ensures the combination of virtual address and size is part of the addressable space. - /// - /// Virtual address of the range - /// Size of the range in bytes - /// Throw when the memory region specified outside the addressable space - private void AssertValidAddressAndSize(ulong va, ulong size) - { - if (!ValidateAddressAndSize(va, size)) - { - throw new InvalidMemoryRegionException($"va=0x{va:X16}, size=0x{size:X16}"); - } - } - /// public void Map(ulong va, ulong pa, ulong size, MemoryMapFlags flags) { @@ -140,18 +59,6 @@ namespace Ryujinx.Cpu.AppleHv Tracking.Map(va, size); } - private void PtMap(ulong va, ulong pa, ulong size) - { - while (size != 0) - { - _pageTable.Map(va, pa); - - va += PageSize; - pa += PageSize; - size -= PageSize; - } - } - /// public void MapForeign(ulong va, nuint hostPointer, ulong size) { @@ -171,544 +78,6 @@ namespace Ryujinx.Cpu.AppleHv PtUnmap(va, size); } - private void PtUnmap(ulong va, ulong size) - { - while (size != 0) - { - _pageTable.Unmap(va); - - va += PageSize; - size -= PageSize; - } - } - - /// - public void Reprotect(ulong va, ulong size, MemoryPermission permission) - { - } - - /// - public T Read(ulong va) where T : unmanaged - { - return MemoryMarshal.Cast(GetSpan(va, Unsafe.SizeOf()))[0]; - } - - /// - public T ReadTracked(ulong va) where T : unmanaged - { - try - { - SignalMemoryTracking(va, (ulong)Unsafe.SizeOf(), false); - - return Read(va); - } - catch (InvalidMemoryRegionException) - { - if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) - { - throw; - } - - return default; - } - } - - /// - public void Read(ulong va, Span data) - { - ReadImpl(va, data); - } - - /// - public void Write(ulong va, T value) where T : unmanaged - { - Write(va, MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref value, 1))); - } - - /// - public void Write(ulong va, ReadOnlySpan data) - { - if (data.Length == 0) - { - return; - } - - SignalMemoryTracking(va, (ulong)data.Length, true); - - WriteImpl(va, data); - } - - /// - public void WriteUntracked(ulong va, ReadOnlySpan data) - { - if (data.Length == 0) - { - return; - } - - WriteImpl(va, data); - } - - /// - public bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan data) - { - if (data.Length == 0) - { - return false; - } - - SignalMemoryTracking(va, (ulong)data.Length, false); - - if (IsContiguousAndMapped(va, data.Length)) - { - var target = _backingMemory.GetSpan(GetPhysicalAddressInternal(va), data.Length); - - bool changed = !data.SequenceEqual(target); - - if (changed) - { - data.CopyTo(target); - } - - return changed; - } - else - { - WriteImpl(va, data); - - return true; - } - } - - private void WriteImpl(ulong va, ReadOnlySpan data) - { - try - { - AssertValidAddressAndSize(va, (ulong)data.Length); - - if (IsContiguousAndMapped(va, data.Length)) - { - data.CopyTo(_backingMemory.GetSpan(GetPhysicalAddressInternal(va), data.Length)); - } - else - { - int offset = 0, size; - - if ((va & PageMask) != 0) - { - ulong pa = GetPhysicalAddressChecked(va); - - size = Math.Min(data.Length, PageSize - (int)(va & PageMask)); - - data[..size].CopyTo(_backingMemory.GetSpan(pa, size)); - - offset += size; - } - - for (; offset < data.Length; offset += size) - { - ulong pa = GetPhysicalAddressChecked(va + (ulong)offset); - - size = Math.Min(data.Length - offset, PageSize); - - data.Slice(offset, size).CopyTo(_backingMemory.GetSpan(pa, size)); - } - } - } - catch (InvalidMemoryRegionException) - { - if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) - { - throw; - } - } - } - - /// - public ReadOnlySpan GetSpan(ulong va, int size, bool tracked = false) - { - if (size == 0) - { - return ReadOnlySpan.Empty; - } - - if (tracked) - { - SignalMemoryTracking(va, (ulong)size, false); - } - - if (IsContiguousAndMapped(va, size)) - { - return _backingMemory.GetSpan(GetPhysicalAddressInternal(va), size); - } - else - { - Span data = new byte[size]; - - ReadImpl(va, data); - - return data; - } - } - - /// - public WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false) - { - if (size == 0) - { - return new WritableRegion(null, va, Memory.Empty); - } - - if (tracked) - { - SignalMemoryTracking(va, (ulong)size, true); - } - - if (IsContiguousAndMapped(va, size)) - { - return new WritableRegion(null, va, _backingMemory.GetMemory(GetPhysicalAddressInternal(va), size)); - } - else - { - Memory memory = new byte[size]; - - ReadImpl(va, memory.Span); - - return new WritableRegion(this, va, memory); - } - } - - /// - public ref T GetRef(ulong va) where T : unmanaged - { - if (!IsContiguous(va, Unsafe.SizeOf())) - { - ThrowMemoryNotContiguous(); - } - - SignalMemoryTracking(va, (ulong)Unsafe.SizeOf(), true); - - return ref _backingMemory.GetRef(GetPhysicalAddressChecked(va)); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool IsMapped(ulong va) - { - return ValidateAddress(va) && IsMappedImpl(va); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool IsMappedImpl(ulong va) - { - ulong page = va >> PageBits; - - int bit = (int)((page & 31) << 1); - - int pageIndex = (int)(page >> PageToPteShift); - ref ulong pageRef = ref _pageBitmap[pageIndex]; - - ulong pte = Volatile.Read(ref pageRef); - - return ((pte >> bit) & 3) != 0; - } - - /// - public bool IsRangeMapped(ulong va, ulong size) - { - AssertValidAddressAndSize(va, size); - - return IsRangeMappedImpl(va, size); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void GetPageBlockRange(ulong pageStart, ulong pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex) - { - startMask = ulong.MaxValue << ((int)(pageStart & 31) << 1); - endMask = ulong.MaxValue >> (64 - ((int)(pageEnd & 31) << 1)); - - pageIndex = (int)(pageStart >> PageToPteShift); - pageEndIndex = (int)((pageEnd - 1) >> PageToPteShift); - } - - private bool IsRangeMappedImpl(ulong va, ulong size) - { - int pages = GetPagesCount(va, size, out _); - - if (pages == 1) - { - return IsMappedImpl(va); - } - - ulong pageStart = va >> PageBits; - ulong pageEnd = pageStart + (ulong)pages; - - GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); - - // Check if either bit in each 2 bit page entry is set. - // OR the block with itself shifted down by 1, and check the first bit of each entry. - - ulong mask = BlockMappedMask & startMask; - - while (pageIndex <= pageEndIndex) - { - if (pageIndex == pageEndIndex) - { - mask &= endMask; - } - - ref ulong pageRef = ref _pageBitmap[pageIndex++]; - ulong pte = Volatile.Read(ref pageRef); - - pte |= pte >> 1; - if ((pte & mask) != mask) - { - return false; - } - - mask = BlockMappedMask; - } - - return true; - } - - private static void ThrowMemoryNotContiguous() => throw new MemoryNotContiguousException(); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool IsContiguousAndMapped(ulong va, int size) => IsContiguous(va, size) && IsMapped(va); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool IsContiguous(ulong va, int size) - { - if (!ValidateAddress(va) || !ValidateAddressAndSize(va, (ulong)size)) - { - return false; - } - - int pages = GetPagesCount(va, (uint)size, out va); - - for (int page = 0; page < pages - 1; page++) - { - if (!ValidateAddress(va + PageSize)) - { - return false; - } - - if (GetPhysicalAddressInternal(va) + PageSize != GetPhysicalAddressInternal(va + PageSize)) - { - return false; - } - - va += PageSize; - } - - return true; - } - - /// - public IEnumerable GetHostRegions(ulong va, ulong size) - { - if (size == 0) - { - return Enumerable.Empty(); - } - - var guestRegions = GetPhysicalRegionsImpl(va, size); - if (guestRegions == null) - { - return null; - } - - var regions = new HostMemoryRange[guestRegions.Count]; - - for (int i = 0; i < regions.Length; i++) - { - var guestRegion = guestRegions[i]; - IntPtr pointer = _backingMemory.GetPointer(guestRegion.Address, guestRegion.Size); - regions[i] = new HostMemoryRange((nuint)(ulong)pointer, guestRegion.Size); - } - - return regions; - } - - /// - public IEnumerable GetPhysicalRegions(ulong va, ulong size) - { - if (size == 0) - { - return Enumerable.Empty(); - } - - return GetPhysicalRegionsImpl(va, size); - } - - private List GetPhysicalRegionsImpl(ulong va, ulong size) - { - if (!ValidateAddress(va) || !ValidateAddressAndSize(va, size)) - { - return null; - } - - int pages = GetPagesCount(va, (uint)size, out va); - - var regions = new List(); - - ulong regionStart = GetPhysicalAddressInternal(va); - ulong regionSize = PageSize; - - for (int page = 0; page < pages - 1; page++) - { - if (!ValidateAddress(va + PageSize)) - { - return null; - } - - ulong newPa = GetPhysicalAddressInternal(va + PageSize); - - if (GetPhysicalAddressInternal(va) + PageSize != newPa) - { - regions.Add(new MemoryRange(regionStart, regionSize)); - regionStart = newPa; - regionSize = 0; - } - - va += PageSize; - regionSize += PageSize; - } - - regions.Add(new MemoryRange(regionStart, regionSize)); - - return regions; - } - - private void ReadImpl(ulong va, Span data) - { - if (data.Length == 0) - { - return; - } - - try - { - AssertValidAddressAndSize(va, (ulong)data.Length); - - int offset = 0, size; - - if ((va & PageMask) != 0) - { - ulong pa = GetPhysicalAddressChecked(va); - - size = Math.Min(data.Length, PageSize - (int)(va & PageMask)); - - _backingMemory.GetSpan(pa, size).CopyTo(data[..size]); - - offset += size; - } - - for (; offset < data.Length; offset += size) - { - ulong pa = GetPhysicalAddressChecked(va + (ulong)offset); - - size = Math.Min(data.Length - offset, PageSize); - - _backingMemory.GetSpan(pa, size).CopyTo(data.Slice(offset, size)); - } - } - catch (InvalidMemoryRegionException) - { - if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) - { - throw; - } - } - } - - /// - /// - /// This function also validates that the given range is both valid and mapped, and will throw if it is not. - /// - public void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null) - { - AssertValidAddressAndSize(va, size); - - if (precise) - { - Tracking.VirtualMemoryEvent(va, size, write, precise: true, exemptId); - return; - } - - // Software table, used for managed memory tracking. - - int pages = GetPagesCount(va, size, out _); - ulong pageStart = va >> PageBits; - - if (pages == 1) - { - ulong tag = (ulong)(write ? HostMappedPtBits.WriteTracked : HostMappedPtBits.ReadWriteTracked); - - int bit = (int)((pageStart & 31) << 1); - - int pageIndex = (int)(pageStart >> PageToPteShift); - ref ulong pageRef = ref _pageBitmap[pageIndex]; - - ulong pte = Volatile.Read(ref pageRef); - ulong state = ((pte >> bit) & 3); - - if (state >= tag) - { - Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId); - return; - } - else if (state == 0) - { - ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}"); - } - } - else - { - ulong pageEnd = pageStart + (ulong)pages; - - GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); - - ulong mask = startMask; - - ulong anyTrackingTag = (ulong)HostMappedPtBits.WriteTrackedReplicated; - - while (pageIndex <= pageEndIndex) - { - if (pageIndex == pageEndIndex) - { - mask &= endMask; - } - - ref ulong pageRef = ref _pageBitmap[pageIndex++]; - - ulong pte = Volatile.Read(ref pageRef); - ulong mappedMask = mask & BlockMappedMask; - - ulong mappedPte = pte | (pte >> 1); - if ((mappedPte & mappedMask) != mappedMask) - { - ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}"); - } - - pte &= mask; - if ((pte & anyTrackingTag) != 0) // Search for any tracking. - { - // Writes trigger any tracking. - // Only trigger tracking from reads if both bits are set on any page. - if (write || (pte & (pte >> 1) & BlockMappedMask) != 0) - { - Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId); - break; - } - } - - mask = ulong.MaxValue; - } - } - } - /// /// Computes the number of pages in a virtual address range. /// @@ -717,7 +86,7 @@ namespace Ryujinx.Cpu.AppleHv /// The virtual address of the beginning of the first page /// This function does not differentiate between allocated and unallocated pages. [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int GetPagesCount(ulong va, ulong size, out ulong startVa) + private int GetPagesCount(ulong va, ulong size, out ulong startVa) { // WARNING: Always check if ulong does not overflow during the operations. startVa = va & ~(ulong)PageMask; @@ -733,206 +102,13 @@ namespace Ryujinx.Cpu.AppleHv } /// - public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection) + public override void TrackingReprotect(ulong va, ulong size, MemoryPermission protection) { - // Protection is inverted on software pages, since the default value is 0. - protection = (~protection) & MemoryPermission.ReadAndWrite; - - int pages = GetPagesCount(va, size, out va); - ulong pageStart = va >> PageBits; - - if (pages == 1) - { - ulong protTag = protection switch - { - MemoryPermission.None => (ulong)HostMappedPtBits.Mapped, - MemoryPermission.Write => (ulong)HostMappedPtBits.WriteTracked, - _ => (ulong)HostMappedPtBits.ReadWriteTracked, - }; - - int bit = (int)((pageStart & 31) << 1); - - ulong tagMask = 3UL << bit; - ulong invTagMask = ~tagMask; - - ulong tag = protTag << bit; - - int pageIndex = (int)(pageStart >> PageToPteShift); - ref ulong pageRef = ref _pageBitmap[pageIndex]; - - ulong pte; - - do - { - pte = Volatile.Read(ref pageRef); - } - while ((pte & tagMask) != 0 && Interlocked.CompareExchange(ref pageRef, (pte & invTagMask) | tag, pte) != pte); - } - else - { - ulong pageEnd = pageStart + (ulong)pages; - - GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); - - ulong mask = startMask; - - ulong protTag = protection switch - { - MemoryPermission.None => (ulong)HostMappedPtBits.MappedReplicated, - MemoryPermission.Write => (ulong)HostMappedPtBits.WriteTrackedReplicated, - _ => (ulong)HostMappedPtBits.ReadWriteTrackedReplicated, - }; - - while (pageIndex <= pageEndIndex) - { - if (pageIndex == pageEndIndex) - { - mask &= endMask; - } - - ref ulong pageRef = ref _pageBitmap[pageIndex++]; - - ulong pte; - ulong mappedMask; - - // Change the protection of all 2 bit entries that are mapped. - do - { - pte = Volatile.Read(ref pageRef); - - mappedMask = pte | (pte >> 1); - mappedMask |= (mappedMask & BlockMappedMask) << 1; - mappedMask &= mask; // Only update mapped pages within the given range. - } - while (Interlocked.CompareExchange(ref pageRef, (pte & (~mappedMask)) | (protTag & mappedMask), pte) != pte); - - mask = ulong.MaxValue; - } - } - - protection = protection switch - { - MemoryPermission.None => MemoryPermission.ReadAndWrite, - MemoryPermission.Write => MemoryPermission.Read, - _ => MemoryPermission.None, - }; + base.TrackingReprotect(va, size, protection); _addressSpace.ReprotectUser(va, size, protection); } - /// - public RegionHandle BeginTracking(ulong address, ulong size, int id) - { - return Tracking.BeginTracking(address, size, id); - } - - /// - public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable handles, ulong granularity, int id) - { - return Tracking.BeginGranularTracking(address, size, handles, granularity, id); - } - - /// - public SmartMultiRegionHandle BeginSmartGranularTracking(ulong address, ulong size, ulong granularity, int id) - { - return Tracking.BeginSmartGranularTracking(address, size, granularity, id); - } - - /// - /// Adds the given address mapping to the page table. - /// - /// Virtual memory address - /// Size to be mapped - private void AddMapping(ulong va, ulong size) - { - int pages = GetPagesCount(va, size, out _); - ulong pageStart = va >> PageBits; - ulong pageEnd = pageStart + (ulong)pages; - - GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); - - ulong mask = startMask; - - while (pageIndex <= pageEndIndex) - { - if (pageIndex == pageEndIndex) - { - mask &= endMask; - } - - ref ulong pageRef = ref _pageBitmap[pageIndex++]; - - ulong pte; - ulong mappedMask; - - // Map all 2-bit entries that are unmapped. - do - { - pte = Volatile.Read(ref pageRef); - - mappedMask = pte | (pte >> 1); - mappedMask |= (mappedMask & BlockMappedMask) << 1; - mappedMask |= ~mask; // Treat everything outside the range as mapped, thus unchanged. - } - while (Interlocked.CompareExchange(ref pageRef, (pte & mappedMask) | (BlockMappedMask & (~mappedMask)), pte) != pte); - - mask = ulong.MaxValue; - } - } - - /// - /// Removes the given address mapping from the page table. - /// - /// Virtual memory address - /// Size to be unmapped - private void RemoveMapping(ulong va, ulong size) - { - int pages = GetPagesCount(va, size, out _); - ulong pageStart = va >> PageBits; - ulong pageEnd = pageStart + (ulong)pages; - - GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); - - startMask = ~startMask; - endMask = ~endMask; - - ulong mask = startMask; - - while (pageIndex <= pageEndIndex) - { - if (pageIndex == pageEndIndex) - { - mask |= endMask; - } - - ref ulong pageRef = ref _pageBitmap[pageIndex++]; - ulong pte; - - do - { - pte = Volatile.Read(ref pageRef); - } - while (Interlocked.CompareExchange(ref pageRef, pte & mask, pte) != pte); - - mask = 0; - } - } - - private ulong GetPhysicalAddressChecked(ulong va) - { - if (!IsMapped(va)) - { - ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}"); - } - - return GetPhysicalAddressInternal(va); - } - - private ulong GetPhysicalAddressInternal(ulong va) - { - return _pageTable.Read(va) + (va & PageMask); - } - /// /// Disposes of resources used by the memory manager. /// diff --git a/src/Ryujinx.Cpu/Jit/MemoryManagerHostNoMirror.cs b/src/Ryujinx.Cpu/Jit/MemoryManagerHostNoMirror.cs new file mode 100644 index 000000000..6fc2f1ef3 --- /dev/null +++ b/src/Ryujinx.Cpu/Jit/MemoryManagerHostNoMirror.cs @@ -0,0 +1,111 @@ +using ARMeilleure.Memory; +using Ryujinx.Memory; +using Ryujinx.Memory.Tracking; +using System; + +namespace Ryujinx.Cpu.Jit +{ + /// + /// Represents a CPU memory manager which maps guest virtual memory directly onto a host virtual region. + /// + public sealed class MemoryManagerHostNoMirror : MemoryManagerSoftware, ICpuMemoryManager, IVirtualMemoryManagerTracked, IWritableBlock + { + private readonly InvalidAccessHandler _invalidAccessHandler; + private readonly bool _unsafeMode; + + private readonly MemoryBlock _addressSpace; + private readonly MemoryBlock _backingMemory; + + public ulong AddressSpaceSize { get; } + + private readonly MemoryEhMeilleure _memoryEh; + + private readonly ulong[] _pageBitmap; + + /// + public bool Supports4KBPages => MemoryBlock.GetPageSize() == PageSize; + + public IntPtr PageTablePointer => _addressSpace.Pointer; + + public MemoryManagerType Type => _unsafeMode ? MemoryManagerType.HostMappedUnsafe : MemoryManagerType.HostMapped; + + public event Action UnmapEvent; + + /// + /// Creates a new instance of the host mapped memory manager. + /// + /// Address space instance to use + /// True if unmanaged access should not be masked (unsafe), false otherwise. + /// Optional function to handle invalid memory accesses + public MemoryManagerHostNoMirror( + MemoryBlock addressSpace, + MemoryBlock backingMemory, + bool unsafeMode, + InvalidAccessHandler invalidAccessHandler) : base(backingMemory, addressSpace.Size, invalidAccessHandler) + { + _addressSpace = addressSpace; + _backingMemory = backingMemory; + _invalidAccessHandler = invalidAccessHandler; + _unsafeMode = unsafeMode; + AddressSpaceSize = addressSpace.Size; + + _pageBitmap = new ulong[1 << (AddressSpaceBits - (PageBits + PageToPteShift))]; + + Tracking = new MemoryTracking(this, (int)MemoryBlock.GetPageSize(), invalidAccessHandler); + _memoryEh = new MemoryEhMeilleure(addressSpace, null, Tracking); + } + + /// + public void Map(ulong va, ulong pa, ulong size, MemoryMapFlags flags) + { + AssertValidAddressAndSize(va, size); + + _addressSpace.MapView(_backingMemory, pa, va, size); + AddMapping(va, size); + PtMap(va, pa, size); + + Tracking.Map(va, size); + } + + /// + public void MapForeign(ulong va, nuint hostPointer, ulong size) + { + throw new NotSupportedException(); + } + + /// + public void Unmap(ulong va, ulong size) + { + AssertValidAddressAndSize(va, size); + + UnmapEvent?.Invoke(va, size); + Tracking.Unmap(va, size); + + RemoveMapping(va, size); + PtUnmap(va, size); + _addressSpace.UnmapView(_backingMemory, va, size); + } + + /// + public void Reprotect(ulong va, ulong size, MemoryPermission permission) + { + } + + /// + public override void TrackingReprotect(ulong va, ulong size, MemoryPermission protection) + { + base.TrackingReprotect(va, size, protection); + + _addressSpace.Reprotect(va, size, protection, false); + } + + /// + /// Disposes of resources used by the memory manager. + /// + protected override void Destroy() + { + _addressSpace.Dispose(); + _memoryEh.Dispose(); + } + } +} diff --git a/src/Ryujinx.Cpu/MemoryEhMeilleure.cs b/src/Ryujinx.Cpu/MemoryEhMeilleure.cs index 74f62bc13..f280614c4 100644 --- a/src/Ryujinx.Cpu/MemoryEhMeilleure.cs +++ b/src/Ryujinx.Cpu/MemoryEhMeilleure.cs @@ -28,7 +28,7 @@ namespace Ryujinx.Cpu throw new InvalidOperationException("Number of allowed tracked regions exceeded."); } - if (OperatingSystem.IsWindows()) + if (OperatingSystem.IsWindows() && addressSpaceMirror != null) { // Add a tracking event with no signal handler for the mirror on Windows. // The native handler has its own code to check for the partial overlap race when regions are protected by accident, diff --git a/src/Ryujinx.Cpu/MemoryManagerSoftware.cs b/src/Ryujinx.Cpu/MemoryManagerSoftware.cs new file mode 100644 index 000000000..427d78ac3 --- /dev/null +++ b/src/Ryujinx.Cpu/MemoryManagerSoftware.cs @@ -0,0 +1,858 @@ +using Ryujinx.Memory; +using Ryujinx.Memory.Range; +using Ryujinx.Memory.Tracking; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Threading; + +namespace Ryujinx.Cpu +{ + /// + /// Represents a CPU memory manager which maps guest virtual memory directly onto the Hypervisor page table. + /// + public class MemoryManagerSoftware : MemoryManagerBase, IWritableBlock + { + public const int PageBits = 12; + public const int PageSize = 1 << PageBits; + public const int PageMask = PageSize - 1; + + public const int PageToPteShift = 5; // 32 pages (2 bits each) in one ulong page table entry. + public const ulong BlockMappedMask = 0x5555555555555555; // First bit of each table entry set. + + private enum HostMappedPtBits : ulong + { + Unmapped = 0, + Mapped, + WriteTracked, + ReadWriteTracked, + + MappedReplicated = 0x5555555555555555, + WriteTrackedReplicated = 0xaaaaaaaaaaaaaaaa, + ReadWriteTrackedReplicated = ulong.MaxValue + } + + private readonly InvalidAccessHandler _invalidAccessHandler; + + private readonly ulong _addressSpaceSize; + + private readonly MemoryBlock _backingMemory; + private readonly PageTable _pageTable; + + private readonly ulong[] _pageBitmap; + + public int AddressSpaceBits { get; } + + public MemoryTracking Tracking { get; protected set; } + + /// + /// Creates a new instance of the Hypervisor memory manager. + /// + /// Physical backing memory where virtual memory will be mapped to + /// Size of the address space + /// Optional function to handle invalid memory accesses + public MemoryManagerSoftware(MemoryBlock backingMemory, ulong addressSpaceSize, InvalidAccessHandler invalidAccessHandler = null) + { + _backingMemory = backingMemory; + _pageTable = new PageTable(); + _invalidAccessHandler = invalidAccessHandler; + _addressSpaceSize = addressSpaceSize; + + ulong asSize = PageSize; + int asBits = PageBits; + + while (asSize < addressSpaceSize) + { + asSize <<= 1; + asBits++; + } + + AddressSpaceBits = asBits; + + _pageBitmap = new ulong[1 << (AddressSpaceBits - (PageBits + PageToPteShift))]; + } + + /// + /// Checks if the virtual address is part of the addressable space. + /// + /// Virtual address + /// True if the virtual address is part of the addressable space + private bool ValidateAddress(ulong va) + { + return va < _addressSpaceSize; + } + + /// + /// Checks if the combination of virtual address and size is part of the addressable space. + /// + /// Virtual address of the range + /// Size of the range in bytes + /// True if the combination of virtual address and size is part of the addressable space + private bool ValidateAddressAndSize(ulong va, ulong size) + { + ulong endVa = va + size; + return endVa >= va && endVa >= size && endVa <= _addressSpaceSize; + } + + /// + /// Ensures the combination of virtual address and size is part of the addressable space. + /// + /// Virtual address of the range + /// Size of the range in bytes + /// Throw when the memory region specified outside the addressable space + protected void AssertValidAddressAndSize(ulong va, ulong size) + { + if (!ValidateAddressAndSize(va, size)) + { + throw new InvalidMemoryRegionException($"va=0x{va:X16}, size=0x{size:X16}"); + } + } + + protected void PtMap(ulong va, ulong pa, ulong size) + { + while (size != 0) + { + _pageTable.Map(va, pa); + + va += PageSize; + pa += PageSize; + size -= PageSize; + } + } + + protected void PtUnmap(ulong va, ulong size) + { + while (size != 0) + { + _pageTable.Unmap(va); + + va += PageSize; + size -= PageSize; + } + } + + public T Read(ulong va) where T : unmanaged + { + return MemoryMarshal.Cast(GetSpan(va, Unsafe.SizeOf()))[0]; + } + + public T ReadTracked(ulong va) where T : unmanaged + { + try + { + SignalMemoryTracking(va, (ulong)Unsafe.SizeOf(), false); + + return Read(va); + } + catch (InvalidMemoryRegionException) + { + if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) + { + throw; + } + + return default; + } + } + + public void Read(ulong va, Span data) + { + ReadImpl(va, data); + } + + public void Write(ulong va, T value) where T : unmanaged + { + Write(va, MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref value, 1))); + } + + public void Write(ulong va, ReadOnlySpan data) + { + if (data.Length == 0) + { + return; + } + + SignalMemoryTracking(va, (ulong)data.Length, true); + + WriteImpl(va, data); + } + + public void WriteUntracked(ulong va, ReadOnlySpan data) + { + if (data.Length == 0) + { + return; + } + + WriteImpl(va, data); + } + + public bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan data) + { + if (data.Length == 0) + { + return false; + } + + SignalMemoryTracking(va, (ulong)data.Length, false); + + if (IsContiguousAndMapped(va, data.Length)) + { + var target = _backingMemory.GetSpan(GetPhysicalAddressInternal(va), data.Length); + + bool changed = !data.SequenceEqual(target); + + if (changed) + { + data.CopyTo(target); + } + + return changed; + } + else + { + WriteImpl(va, data); + + return true; + } + } + + private void WriteImpl(ulong va, ReadOnlySpan data) + { + try + { + AssertValidAddressAndSize(va, (ulong)data.Length); + + if (IsContiguousAndMapped(va, data.Length)) + { + data.CopyTo(_backingMemory.GetSpan(GetPhysicalAddressInternal(va), data.Length)); + } + else + { + int offset = 0, size; + + if ((va & PageMask) != 0) + { + ulong pa = GetPhysicalAddressChecked(va); + + size = Math.Min(data.Length, PageSize - (int)(va & PageMask)); + + data.Slice(0, size).CopyTo(_backingMemory.GetSpan(pa, size)); + + offset += size; + } + + for (; offset < data.Length; offset += size) + { + ulong pa = GetPhysicalAddressChecked(va + (ulong)offset); + + size = Math.Min(data.Length - offset, PageSize); + + data.Slice(offset, size).CopyTo(_backingMemory.GetSpan(pa, size)); + } + } + } + catch (InvalidMemoryRegionException) + { + if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) + { + throw; + } + } + } + + public ReadOnlySpan GetSpan(ulong va, int size, bool tracked = false) + { + if (size == 0) + { + return ReadOnlySpan.Empty; + } + + if (tracked) + { + SignalMemoryTracking(va, (ulong)size, false); + } + + if (IsContiguousAndMapped(va, size)) + { + return _backingMemory.GetSpan(GetPhysicalAddressInternal(va), size); + } + else + { + Span data = new byte[size]; + + ReadImpl(va, data); + + return data; + } + } + + public WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false) + { + if (size == 0) + { + return new WritableRegion(null, va, Memory.Empty); + } + + if (tracked) + { + SignalMemoryTracking(va, (ulong)size, true); + } + + if (IsContiguousAndMapped(va, size)) + { + return new WritableRegion(null, va, _backingMemory.GetMemory(GetPhysicalAddressInternal(va), size)); + } + else + { + Memory memory = new byte[size]; + + ReadImpl(va, memory.Span); + + return new WritableRegion(this, va, memory); + } + } + + public ref T GetRef(ulong va) where T : unmanaged + { + if (!IsContiguous(va, Unsafe.SizeOf())) + { + ThrowMemoryNotContiguous(); + } + + SignalMemoryTracking(va, (ulong)Unsafe.SizeOf(), true); + + return ref _backingMemory.GetRef(GetPhysicalAddressChecked(va)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool IsMapped(ulong va) + { + return ValidateAddress(va) && IsMappedImpl(va); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool IsMappedImpl(ulong va) + { + ulong page = va >> PageBits; + + int bit = (int)((page & 31) << 1); + + int pageIndex = (int)(page >> PageToPteShift); + ref ulong pageRef = ref _pageBitmap[pageIndex]; + + ulong pte = Volatile.Read(ref pageRef); + + return ((pte >> bit) & 3) != 0; + } + + public bool IsRangeMapped(ulong va, ulong size) + { + AssertValidAddressAndSize(va, size); + + return IsRangeMappedImpl(va, size); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void GetPageBlockRange(ulong pageStart, ulong pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex) + { + startMask = ulong.MaxValue << ((int)(pageStart & 31) << 1); + endMask = ulong.MaxValue >> (64 - ((int)(pageEnd & 31) << 1)); + + pageIndex = (int)(pageStart >> PageToPteShift); + pageEndIndex = (int)((pageEnd - 1) >> PageToPteShift); + } + + private bool IsRangeMappedImpl(ulong va, ulong size) + { + int pages = GetPagesCount(va, size, out _); + + if (pages == 1) + { + return IsMappedImpl(va); + } + + ulong pageStart = va >> PageBits; + ulong pageEnd = pageStart + (ulong)pages; + + GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); + + // Check if either bit in each 2 bit page entry is set. + // OR the block with itself shifted down by 1, and check the first bit of each entry. + + ulong mask = BlockMappedMask & startMask; + + while (pageIndex <= pageEndIndex) + { + if (pageIndex == pageEndIndex) + { + mask &= endMask; + } + + ref ulong pageRef = ref _pageBitmap[pageIndex++]; + ulong pte = Volatile.Read(ref pageRef); + + pte |= pte >> 1; + if ((pte & mask) != mask) + { + return false; + } + + mask = BlockMappedMask; + } + + return true; + } + + private static void ThrowMemoryNotContiguous() => throw new MemoryNotContiguousException(); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool IsContiguousAndMapped(ulong va, int size) => IsContiguous(va, size) && IsMapped(va); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool IsContiguous(ulong va, int size) + { + if (!ValidateAddress(va) || !ValidateAddressAndSize(va, (ulong)size)) + { + return false; + } + + int pages = GetPagesCount(va, (uint)size, out va); + + for (int page = 0; page < pages - 1; page++) + { + if (!ValidateAddress(va + PageSize)) + { + return false; + } + + if (GetPhysicalAddressInternal(va) + PageSize != GetPhysicalAddressInternal(va + PageSize)) + { + return false; + } + + va += PageSize; + } + + return true; + } + + public IEnumerable GetHostRegions(ulong va, ulong size) + { + if (size == 0) + { + return Enumerable.Empty(); + } + + var guestRegions = GetPhysicalRegionsImpl(va, size); + if (guestRegions == null) + { + return null; + } + + var regions = new HostMemoryRange[guestRegions.Count]; + + for (int i = 0; i < regions.Length; i++) + { + var guestRegion = guestRegions[i]; + IntPtr pointer = _backingMemory.GetPointer(guestRegion.Address, guestRegion.Size); + regions[i] = new HostMemoryRange((nuint)(ulong)pointer, guestRegion.Size); + } + + return regions; + } + + public IEnumerable GetPhysicalRegions(ulong va, ulong size) + { + if (size == 0) + { + return Enumerable.Empty(); + } + + return GetPhysicalRegionsImpl(va, size); + } + + private List GetPhysicalRegionsImpl(ulong va, ulong size) + { + if (!ValidateAddress(va) || !ValidateAddressAndSize(va, size)) + { + return null; + } + + int pages = GetPagesCount(va, (uint)size, out va); + + var regions = new List(); + + ulong regionStart = GetPhysicalAddressInternal(va); + ulong regionSize = PageSize; + + for (int page = 0; page < pages - 1; page++) + { + if (!ValidateAddress(va + PageSize)) + { + return null; + } + + ulong newPa = GetPhysicalAddressInternal(va + PageSize); + + if (GetPhysicalAddressInternal(va) + PageSize != newPa) + { + regions.Add(new MemoryRange(regionStart, regionSize)); + regionStart = newPa; + regionSize = 0; + } + + va += PageSize; + regionSize += PageSize; + } + + regions.Add(new MemoryRange(regionStart, regionSize)); + + return regions; + } + + private void ReadImpl(ulong va, Span data) + { + if (data.Length == 0) + { + return; + } + + try + { + AssertValidAddressAndSize(va, (ulong)data.Length); + + int offset = 0, size; + + if ((va & PageMask) != 0) + { + ulong pa = GetPhysicalAddressChecked(va); + + size = Math.Min(data.Length, PageSize - (int)(va & PageMask)); + + _backingMemory.GetSpan(pa, size).CopyTo(data.Slice(0, size)); + + offset += size; + } + + for (; offset < data.Length; offset += size) + { + ulong pa = GetPhysicalAddressChecked(va + (ulong)offset); + + size = Math.Min(data.Length - offset, PageSize); + + _backingMemory.GetSpan(pa, size).CopyTo(data.Slice(offset, size)); + } + } + catch (InvalidMemoryRegionException) + { + if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) + { + throw; + } + } + } + + /// + /// + /// This function also validates that the given range is both valid and mapped, and will throw if it is not. + /// + public void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null) + { + AssertValidAddressAndSize(va, size); + + if (precise) + { + Tracking.VirtualMemoryEvent(va, size, write, precise: true, exemptId); + return; + } + + // Software table, used for managed memory tracking. + + int pages = GetPagesCount(va, size, out _); + ulong pageStart = va >> PageBits; + + if (pages == 1) + { + ulong tag = (ulong)(write ? HostMappedPtBits.WriteTracked : HostMappedPtBits.ReadWriteTracked); + + int bit = (int)((pageStart & 31) << 1); + + int pageIndex = (int)(pageStart >> PageToPteShift); + ref ulong pageRef = ref _pageBitmap[pageIndex]; + + ulong pte = Volatile.Read(ref pageRef); + ulong state = ((pte >> bit) & 3); + + if (state >= tag) + { + Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId); + return; + } + else if (state == 0) + { + ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}"); + } + } + else + { + ulong pageEnd = pageStart + (ulong)pages; + + GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); + + ulong mask = startMask; + + ulong anyTrackingTag = (ulong)HostMappedPtBits.WriteTrackedReplicated; + + while (pageIndex <= pageEndIndex) + { + if (pageIndex == pageEndIndex) + { + mask &= endMask; + } + + ref ulong pageRef = ref _pageBitmap[pageIndex++]; + + ulong pte = Volatile.Read(ref pageRef); + ulong mappedMask = mask & BlockMappedMask; + + ulong mappedPte = pte | (pte >> 1); + if ((mappedPte & mappedMask) != mappedMask) + { + ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}"); + } + + pte &= mask; + if ((pte & anyTrackingTag) != 0) // Search for any tracking. + { + // Writes trigger any tracking. + // Only trigger tracking from reads if both bits are set on any page. + if (write || (pte & (pte >> 1) & BlockMappedMask) != 0) + { + Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId); + break; + } + } + + mask = ulong.MaxValue; + } + } + } + + /// + /// Computes the number of pages in a virtual address range. + /// + /// Virtual address of the range + /// Size of the range + /// The virtual address of the beginning of the first page + /// This function does not differentiate between allocated and unallocated pages. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private int GetPagesCount(ulong va, ulong size, out ulong startVa) + { + // WARNING: Always check if ulong does not overflow during the operations. + startVa = va & ~(ulong)PageMask; + ulong vaSpan = (va - startVa + size + PageMask) & ~(ulong)PageMask; + + return (int)(vaSpan / PageSize); + } + + public RegionHandle BeginTracking(ulong address, ulong size, int id) + { + return Tracking.BeginTracking(address, size, id); + } + + public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable handles, ulong granularity, int id) + { + return Tracking.BeginGranularTracking(address, size, handles, granularity, id); + } + + public SmartMultiRegionHandle BeginSmartGranularTracking(ulong address, ulong size, ulong granularity, int id) + { + return Tracking.BeginSmartGranularTracking(address, size, granularity, id); + } + + public virtual void TrackingReprotect(ulong va, ulong size, MemoryPermission protection) + { + // Protection is inverted on software pages, since the default value is 0. + protection = (~protection) & MemoryPermission.ReadAndWrite; + + int pages = GetPagesCount(va, size, out va); + ulong pageStart = va >> PageBits; + + if (pages == 1) + { + ulong protTag = protection switch + { + MemoryPermission.None => (ulong)HostMappedPtBits.Mapped, + MemoryPermission.Write => (ulong)HostMappedPtBits.WriteTracked, + _ => (ulong)HostMappedPtBits.ReadWriteTracked, + }; + + int bit = (int)((pageStart & 31) << 1); + + ulong tagMask = 3UL << bit; + ulong invTagMask = ~tagMask; + + ulong tag = protTag << bit; + + int pageIndex = (int)(pageStart >> PageToPteShift); + ref ulong pageRef = ref _pageBitmap[pageIndex]; + + ulong pte; + + do + { + pte = Volatile.Read(ref pageRef); + } + while ((pte & tagMask) != 0 && Interlocked.CompareExchange(ref pageRef, (pte & invTagMask) | tag, pte) != pte); + } + else + { + ulong pageEnd = pageStart + (ulong)pages; + + GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); + + ulong mask = startMask; + + ulong protTag = protection switch + { + MemoryPermission.None => (ulong)HostMappedPtBits.MappedReplicated, + MemoryPermission.Write => (ulong)HostMappedPtBits.WriteTrackedReplicated, + _ => (ulong)HostMappedPtBits.ReadWriteTrackedReplicated, + }; + + while (pageIndex <= pageEndIndex) + { + if (pageIndex == pageEndIndex) + { + mask &= endMask; + } + + ref ulong pageRef = ref _pageBitmap[pageIndex++]; + + ulong pte; + ulong mappedMask; + + // Change the protection of all 2 bit entries that are mapped. + do + { + pte = Volatile.Read(ref pageRef); + + mappedMask = pte | (pte >> 1); + mappedMask |= (mappedMask & BlockMappedMask) << 1; + mappedMask &= mask; // Only update mapped pages within the given range. + } + while (Interlocked.CompareExchange(ref pageRef, (pte & (~mappedMask)) | (protTag & mappedMask), pte) != pte); + + mask = ulong.MaxValue; + } + } + } + + /// + /// Adds the given address mapping to the page table. + /// + /// Virtual memory address + /// Size to be mapped + protected void AddMapping(ulong va, ulong size) + { + int pages = GetPagesCount(va, size, out _); + ulong pageStart = va >> PageBits; + ulong pageEnd = pageStart + (ulong)pages; + + GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); + + ulong mask = startMask; + + while (pageIndex <= pageEndIndex) + { + if (pageIndex == pageEndIndex) + { + mask &= endMask; + } + + ref ulong pageRef = ref _pageBitmap[pageIndex++]; + + ulong pte; + ulong mappedMask; + + // Map all 2-bit entries that are unmapped. + do + { + pte = Volatile.Read(ref pageRef); + + mappedMask = pte | (pte >> 1); + mappedMask |= (mappedMask & BlockMappedMask) << 1; + mappedMask |= ~mask; // Treat everything outside the range as mapped, thus unchanged. + } + while (Interlocked.CompareExchange(ref pageRef, (pte & mappedMask) | (BlockMappedMask & (~mappedMask)), pte) != pte); + + mask = ulong.MaxValue; + } + } + + /// + /// Removes the given address mapping from the page table. + /// + /// Virtual memory address + /// Size to be unmapped + protected void RemoveMapping(ulong va, ulong size) + { + int pages = GetPagesCount(va, size, out _); + ulong pageStart = va >> PageBits; + ulong pageEnd = pageStart + (ulong)pages; + + GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); + + startMask = ~startMask; + endMask = ~endMask; + + ulong mask = startMask; + + while (pageIndex <= pageEndIndex) + { + if (pageIndex == pageEndIndex) + { + mask |= endMask; + } + + ref ulong pageRef = ref _pageBitmap[pageIndex++]; + ulong pte; + + do + { + pte = Volatile.Read(ref pageRef); + } + while (Interlocked.CompareExchange(ref pageRef, pte & mask, pte) != pte); + + mask = 0; + } + } + + private ulong GetPhysicalAddressChecked(ulong va) + { + if (!IsMapped(va)) + { + ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}"); + } + + return GetPhysicalAddressInternal(va); + } + + private ulong GetPhysicalAddressInternal(ulong va) + { + return _pageTable.Read(va) + (va & PageMask); + } + + private static void ThrowInvalidMemoryRegionException(string message) => throw new InvalidMemoryRegionException(message); + + /// + /// Disposes of resources used by the memory manager. + /// + protected override void Destroy() + { + } + } +} \ No newline at end of file diff --git a/src/Ryujinx.Cpu/Nce/MemoryManagerNative.cs b/src/Ryujinx.Cpu/Nce/MemoryManagerNative.cs index e7c757247..a816b360e 100644 --- a/src/Ryujinx.Cpu/Nce/MemoryManagerNative.cs +++ b/src/Ryujinx.Cpu/Nce/MemoryManagerNative.cs @@ -1,152 +1,55 @@ using ARMeilleure.Memory; using Ryujinx.Memory; -using Ryujinx.Memory.Range; using Ryujinx.Memory.Tracking; using System; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.CompilerServices; -using System.Threading; namespace Ryujinx.Cpu.Nce { /// /// Represents a CPU memory manager which maps guest virtual memory directly onto a host virtual region. /// - public sealed class MemoryManagerNative : MemoryManagerBase, ICpuMemoryManager, IVirtualMemoryManagerTracked, IWritableBlock + public sealed class MemoryManagerNative : MemoryManagerSoftware, ICpuMemoryManager, IVirtualMemoryManagerTracked, IWritableBlock { - public const int PageBits = 12; - public const int PageSize = 1 << PageBits; - public const int PageMask = PageSize - 1; - - public const int PageToPteShift = 5; // 32 pages (2 bits each) in one ulong page table entry. - public const ulong BlockMappedMask = 0x5555555555555555; // First bit of each table entry set. - - private enum HostMappedPtBits : ulong - { - Unmapped = 0, - Mapped, - WriteTracked, - ReadWriteTracked, - - MappedReplicated = 0x5555555555555555, - WriteTrackedReplicated = 0xaaaaaaaaaaaaaaaa, - ReadWriteTrackedReplicated = ulong.MaxValue - } - private readonly InvalidAccessHandler _invalidAccessHandler; private readonly MemoryBlock _addressSpace; - private readonly MemoryBlock _addressSpaceMirror; private readonly ulong _addressSpaceSize; private readonly MemoryBlock _backingMemory; - private readonly PageTable _pageTable; private readonly MemoryEhMeilleure _memoryEh; - private readonly ulong[] _pageBitmap; - /// public bool Supports4KBPages => MemoryBlock.GetPageSize() == PageSize; - public int AddressSpaceBits { get; } - public IntPtr PageTablePointer => IntPtr.Zero; public ulong ReservedSize => (ulong)_addressSpace.Pointer.ToInt64(); public MemoryManagerType Type => MemoryManagerType.HostMappedUnsafe; - public MemoryTracking Tracking { get; } - public event Action UnmapEvent; /// /// Creates a new instance of the host mapped memory manager. /// - /// Address space instance to use + /// Address space memory block /// Physical backing memory where virtual memory will be mapped to /// Size of the address space /// Optional function to handle invalid memory accesses public MemoryManagerNative( - AddressSpace addressSpace, + MemoryBlock addressSpace, MemoryBlock backingMemory, ulong addressSpaceSize, - InvalidAccessHandler invalidAccessHandler = null) + InvalidAccessHandler invalidAccessHandler = null) : base(backingMemory, addressSpaceSize, invalidAccessHandler) { _backingMemory = backingMemory; - _pageTable = new PageTable(); _invalidAccessHandler = invalidAccessHandler; _addressSpaceSize = addressSpaceSize; - - ulong asSize = PageSize; - int asBits = PageBits; - - while (asSize < addressSpaceSize) - { - asSize <<= 1; - asBits++; - } - - AddressSpaceBits = asBits; - - _pageBitmap = new ulong[1 << (AddressSpaceBits - (PageBits + PageToPteShift))]; - - _addressSpace = addressSpace.Base; - _addressSpaceMirror = addressSpace.Mirror; + _addressSpace = addressSpace; Tracking = new MemoryTracking(this, PageSize, invalidAccessHandler); - _memoryEh = new MemoryEhMeilleure(asSize, Tracking); - } - - /// - /// Checks if the virtual address is part of the addressable space. - /// - /// Virtual address - /// True if the virtual address is part of the addressable space - private bool ValidateAddress(ulong va) - { - return va < _addressSpaceSize; - } - - /// - /// Checks if the combination of virtual address and size is part of the addressable space. - /// - /// Virtual address of the range - /// Size of the range in bytes - /// True if the combination of virtual address and size is part of the addressable space - private bool ValidateAddressAndSize(ulong va, ulong size) - { - ulong endVa = va + size; - return endVa >= va && endVa >= size && endVa <= _addressSpaceSize; - } - - /// - /// Ensures the combination of virtual address and size is part of the addressable space. - /// - /// Virtual address of the range - /// Size of the range in bytes - /// Throw when the memory region specified outside the addressable space - private void AssertValidAddressAndSize(ulong va, ulong size) - { - if (!ValidateAddressAndSize(va, size)) - { - throw new InvalidMemoryRegionException($"va=0x{va:X16}, size=0x{size:X16}"); - } - } - - /// - /// Ensures the combination of virtual address and size is part of the addressable space and fully mapped. - /// - /// Virtual address of the range - /// Size of the range in bytes - private void AssertMapped(ulong va, ulong size) - { - if (!ValidateAddressAndSize(va, size) || !IsRangeMappedImpl(va, size)) - { - throw new InvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}"); - } + _memoryEh = new MemoryEhMeilleure(addressSpaceSize, Tracking); } /// @@ -155,7 +58,6 @@ namespace Ryujinx.Cpu.Nce AssertValidAddressAndSize(va, size); _addressSpace.MapView(_backingMemory, pa, AddressToOffset(va), size); - _addressSpaceMirror.MapView(_backingMemory, pa, AddressToOffset(va), size); AddMapping(va, size); PtMap(va, pa, size); @@ -179,30 +81,6 @@ namespace Ryujinx.Cpu.Nce RemoveMapping(va, size); PtUnmap(va, size); _addressSpace.UnmapView(_backingMemory, AddressToOffset(va), size); - _addressSpaceMirror.UnmapView(_backingMemory, AddressToOffset(va), size); - } - - private void PtMap(ulong va, ulong pa, ulong size) - { - while (size != 0) - { - _pageTable.Map(va, pa); - - va += PageSize; - pa += PageSize; - size -= PageSize; - } - } - - private void PtUnmap(ulong va, ulong size) - { - while (size != 0) - { - _pageTable.Unmap(va); - - va += PageSize; - size -= PageSize; - } } /// @@ -212,617 +90,13 @@ namespace Ryujinx.Cpu.Nce } /// - public T Read(ulong va) where T : unmanaged + public override void TrackingReprotect(ulong va, ulong size, MemoryPermission protection) { - try - { - AssertMapped(va, (ulong)Unsafe.SizeOf()); - - return _addressSpaceMirror.Read(AddressToOffset(va)); - } - catch (InvalidMemoryRegionException) - { - if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) - { - throw; - } - - return default; - } - } - - /// - public T ReadTracked(ulong va) where T : unmanaged - { - try - { - SignalMemoryTracking(va, (ulong)Unsafe.SizeOf(), false); - - return Read(va); - } - catch (InvalidMemoryRegionException) - { - if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) - { - throw; - } - - return default; - } - } - - /// - public void Read(ulong va, Span data) - { - try - { - AssertMapped(va, (ulong)data.Length); - - _addressSpaceMirror.Read(AddressToOffset(va), data); - } - catch (InvalidMemoryRegionException) - { - if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) - { - throw; - } - } - } - - - /// - public void Write(ulong va, T value) where T : unmanaged - { - try - { - SignalMemoryTracking(va, (ulong)Unsafe.SizeOf(), write: true); - - _addressSpaceMirror.Write(AddressToOffset(va), value); - } - catch (InvalidMemoryRegionException) - { - if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) - { - throw; - } - } - } - - /// - public void Write(ulong va, ReadOnlySpan data) - { - try - { - SignalMemoryTracking(va, (ulong)data.Length, write: true); - - _addressSpaceMirror.Write(AddressToOffset(va), data); - } - catch (InvalidMemoryRegionException) - { - if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) - { - throw; - } - } - } - - /// - public void WriteUntracked(ulong va, ReadOnlySpan data) - { - try - { - AssertMapped(va, (ulong)data.Length); - - _addressSpaceMirror.Write(AddressToOffset(va), data); - } - catch (InvalidMemoryRegionException) - { - if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) - { - throw; - } - } - } - - /// - public bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan data) - { - try - { - SignalMemoryTracking(va, (ulong)data.Length, false); - - Span target = _addressSpaceMirror.GetSpan(AddressToOffset(va), data.Length); - bool changed = !data.SequenceEqual(target); - - if (changed) - { - data.CopyTo(target); - } - - return changed; - } - catch (InvalidMemoryRegionException) - { - if (_invalidAccessHandler == null || !_invalidAccessHandler(va)) - { - throw; - } - - return true; - } - } - - /// - public ReadOnlySpan GetSpan(ulong va, int size, bool tracked = false) - { - if (tracked) - { - SignalMemoryTracking(va, (ulong)size, write: false); - } - else - { - AssertMapped(va, (ulong)size); - } - - if (size == 0) - { - return ReadOnlySpan.Empty; - } - - return _addressSpaceMirror.GetSpan(AddressToOffset(va), size); - } - - /// - public WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false) - { - if (tracked) - { - SignalMemoryTracking(va, (ulong)size, true); - } - else - { - AssertMapped(va, (ulong)size); - } - - return _addressSpaceMirror.GetWritableRegion(AddressToOffset(va), size); - } - - /// - public ref T GetRef(ulong va) where T : unmanaged - { - SignalMemoryTracking(va, (ulong)Unsafe.SizeOf(), true); - - return ref _addressSpaceMirror.GetRef(AddressToOffset(va)); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool IsMapped(ulong va) - { - return ValidateAddress(va) && IsMappedImpl(va); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool IsMappedImpl(ulong va) - { - ulong page = va >> PageBits; - - int bit = (int)((page & 31) << 1); - - int pageIndex = (int)(page >> PageToPteShift); - ref ulong pageRef = ref _pageBitmap[pageIndex]; - - ulong pte = Volatile.Read(ref pageRef); - - return ((pte >> bit) & 3) != 0; - } - - /// - public bool IsRangeMapped(ulong va, ulong size) - { - AssertValidAddressAndSize(va, size); - - return IsRangeMappedImpl(va, size); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void GetPageBlockRange(ulong pageStart, ulong pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex) - { - startMask = ulong.MaxValue << ((int)(pageStart & 31) << 1); - endMask = ulong.MaxValue >> (64 - ((int)(pageEnd & 31) << 1)); - - pageIndex = (int)(pageStart >> PageToPteShift); - pageEndIndex = (int)((pageEnd - 1) >> PageToPteShift); - } - - private bool IsRangeMappedImpl(ulong va, ulong size) - { - int pages = GetPagesCount(va, size, out _); - - if (pages == 1) - { - return IsMappedImpl(va); - } - - ulong pageStart = va >> PageBits; - ulong pageEnd = pageStart + (ulong)pages; - - GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); - - // Check if either bit in each 2 bit page entry is set. - // OR the block with itself shifted down by 1, and check the first bit of each entry. - - ulong mask = BlockMappedMask & startMask; - - while (pageIndex <= pageEndIndex) - { - if (pageIndex == pageEndIndex) - { - mask &= endMask; - } - - ref ulong pageRef = ref _pageBitmap[pageIndex++]; - ulong pte = Volatile.Read(ref pageRef); - - pte |= pte >> 1; - if ((pte & mask) != mask) - { - return false; - } - - mask = BlockMappedMask; - } - - return true; - } - - /// - public IEnumerable GetHostRegions(ulong va, ulong size) - { - AssertValidAddressAndSize(va, size); - - return Enumerable.Repeat(new HostMemoryRange((nuint)(ulong)_addressSpaceMirror.GetPointer(AddressToOffset(va), size), size), 1); - } - - /// - public IEnumerable GetPhysicalRegions(ulong va, ulong size) - { - int pages = GetPagesCount(va, (uint)size, out va); - - var regions = new List(); - - ulong regionStart = GetPhysicalAddressChecked(va); - ulong regionSize = PageSize; - - for (int page = 0; page < pages - 1; page++) - { - if (!ValidateAddress(va + PageSize)) - { - return null; - } - - ulong newPa = GetPhysicalAddressChecked(va + PageSize); - - if (GetPhysicalAddressChecked(va) + PageSize != newPa) - { - regions.Add(new MemoryRange(regionStart, regionSize)); - regionStart = newPa; - regionSize = 0; - } - - va += PageSize; - regionSize += PageSize; - } - - regions.Add(new MemoryRange(regionStart, regionSize)); - - return regions; - } - - private ulong GetPhysicalAddressChecked(ulong va) - { - if (!IsMapped(va)) - { - ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}"); - } - - return GetPhysicalAddressInternal(va); - } - - private ulong GetPhysicalAddressInternal(ulong va) - { - return _pageTable.Read(va) + (va & PageMask); - } - - /// - /// - /// This function also validates that the given range is both valid and mapped, and will throw if it is not. - /// - public void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null) - { - AssertValidAddressAndSize(va, size); - - if (precise) - { - Tracking.VirtualMemoryEvent(va, size, write, precise: true, exemptId); - return; - } - - // Software table, used for managed memory tracking. - - int pages = GetPagesCount(va, size, out _); - ulong pageStart = va >> PageBits; - - if (pages == 1) - { - ulong tag = (ulong)(write ? HostMappedPtBits.WriteTracked : HostMappedPtBits.ReadWriteTracked); - - int bit = (int)((pageStart & 31) << 1); - - int pageIndex = (int)(pageStart >> PageToPteShift); - ref ulong pageRef = ref _pageBitmap[pageIndex]; - - ulong pte = Volatile.Read(ref pageRef); - ulong state = ((pte >> bit) & 3); - - if (state >= tag) - { - Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId); - return; - } - else if (state == 0) - { - ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}"); - } - } - else - { - ulong pageEnd = pageStart + (ulong)pages; - - GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); - - ulong mask = startMask; - - ulong anyTrackingTag = (ulong)HostMappedPtBits.WriteTrackedReplicated; - - while (pageIndex <= pageEndIndex) - { - if (pageIndex == pageEndIndex) - { - mask &= endMask; - } - - ref ulong pageRef = ref _pageBitmap[pageIndex++]; - - ulong pte = Volatile.Read(ref pageRef); - ulong mappedMask = mask & BlockMappedMask; - - ulong mappedPte = pte | (pte >> 1); - if ((mappedPte & mappedMask) != mappedMask) - { - ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}"); - } - - pte &= mask; - if ((pte & anyTrackingTag) != 0) // Search for any tracking. - { - // Writes trigger any tracking. - // Only trigger tracking from reads if both bits are set on any page. - if (write || (pte & (pte >> 1) & BlockMappedMask) != 0) - { - Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId); - break; - } - } - - mask = ulong.MaxValue; - } - } - } - - /// - /// Computes the number of pages in a virtual address range. - /// - /// Virtual address of the range - /// Size of the range - /// The virtual address of the beginning of the first page - /// This function does not differentiate between allocated and unallocated pages. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private int GetPagesCount(ulong va, ulong size, out ulong startVa) - { - // WARNING: Always check if ulong does not overflow during the operations. - startVa = va & ~(ulong)PageMask; - ulong vaSpan = (va - startVa + size + PageMask) & ~(ulong)PageMask; - - return (int)(vaSpan / PageSize); - } - - /// - public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection) - { - // Protection is inverted on software pages, since the default value is 0. - protection = (~protection) & MemoryPermission.ReadAndWrite; - - int pages = GetPagesCount(va, size, out va); - ulong pageStart = va >> PageBits; - - if (pages == 1) - { - ulong protTag = protection switch - { - MemoryPermission.None => (ulong)HostMappedPtBits.Mapped, - MemoryPermission.Write => (ulong)HostMappedPtBits.WriteTracked, - _ => (ulong)HostMappedPtBits.ReadWriteTracked, - }; - - int bit = (int)((pageStart & 31) << 1); - - ulong tagMask = 3UL << bit; - ulong invTagMask = ~tagMask; - - ulong tag = protTag << bit; - - int pageIndex = (int)(pageStart >> PageToPteShift); - ref ulong pageRef = ref _pageBitmap[pageIndex]; - - ulong pte; - - do - { - pte = Volatile.Read(ref pageRef); - } - while ((pte & tagMask) != 0 && Interlocked.CompareExchange(ref pageRef, (pte & invTagMask) | tag, pte) != pte); - } - else - { - ulong pageEnd = pageStart + (ulong)pages; - - GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); - - ulong mask = startMask; - - ulong protTag = protection switch - { - MemoryPermission.None => (ulong)HostMappedPtBits.MappedReplicated, - MemoryPermission.Write => (ulong)HostMappedPtBits.WriteTrackedReplicated, - _ => (ulong)HostMappedPtBits.ReadWriteTrackedReplicated, - }; - - while (pageIndex <= pageEndIndex) - { - if (pageIndex == pageEndIndex) - { - mask &= endMask; - } - - ref ulong pageRef = ref _pageBitmap[pageIndex++]; - - ulong pte; - ulong mappedMask; - - // Change the protection of all 2 bit entries that are mapped. - do - { - pte = Volatile.Read(ref pageRef); - - mappedMask = pte | (pte >> 1); - mappedMask |= (mappedMask & BlockMappedMask) << 1; - mappedMask &= mask; // Only update mapped pages within the given range. - } - while (Interlocked.CompareExchange(ref pageRef, (pte & (~mappedMask)) | (protTag & mappedMask), pte) != pte); - - mask = ulong.MaxValue; - } - } - - protection = protection switch - { - MemoryPermission.None => MemoryPermission.ReadAndWrite, - MemoryPermission.Write => MemoryPermission.Read, - _ => MemoryPermission.None - }; + base.TrackingReprotect(va, size, protection); _addressSpace.Reprotect(AddressToOffset(va), size, protection, false); } - /// - public RegionHandle BeginTracking(ulong address, ulong size, int id) - { - return Tracking.BeginTracking(address, size, id); - } - - /// - public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable handles, ulong granularity, int id) - { - return Tracking.BeginGranularTracking(address, size, handles, granularity, id); - } - - /// - public SmartMultiRegionHandle BeginSmartGranularTracking(ulong address, ulong size, ulong granularity, int id) - { - return Tracking.BeginSmartGranularTracking(address, size, granularity, id); - } - - /// - /// Adds the given address mapping to the page table. - /// - /// Virtual memory address - /// Size to be mapped - private void AddMapping(ulong va, ulong size) - { - int pages = GetPagesCount(va, size, out _); - ulong pageStart = va >> PageBits; - ulong pageEnd = pageStart + (ulong)pages; - - GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); - - ulong mask = startMask; - - while (pageIndex <= pageEndIndex) - { - if (pageIndex == pageEndIndex) - { - mask &= endMask; - } - - ref ulong pageRef = ref _pageBitmap[pageIndex++]; - - ulong pte; - ulong mappedMask; - - // Map all 2-bit entries that are unmapped. - do - { - pte = Volatile.Read(ref pageRef); - - mappedMask = pte | (pte >> 1); - mappedMask |= (mappedMask & BlockMappedMask) << 1; - mappedMask |= ~mask; // Treat everything outside the range as mapped, thus unchanged. - } - while (Interlocked.CompareExchange(ref pageRef, (pte & mappedMask) | (BlockMappedMask & (~mappedMask)), pte) != pte); - - mask = ulong.MaxValue; - } - } - - /// - /// Removes the given address mapping from the page table. - /// - /// Virtual memory address - /// Size to be unmapped - private void RemoveMapping(ulong va, ulong size) - { - int pages = GetPagesCount(va, size, out _); - ulong pageStart = va >> PageBits; - ulong pageEnd = pageStart + (ulong)pages; - - GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex); - - startMask = ~startMask; - endMask = ~endMask; - - ulong mask = startMask; - - while (pageIndex <= pageEndIndex) - { - if (pageIndex == pageEndIndex) - { - mask |= endMask; - } - - ref ulong pageRef = ref _pageBitmap[pageIndex++]; - ulong pte; - - do - { - pte = Volatile.Read(ref pageRef); - } - while (Interlocked.CompareExchange(ref pageRef, pte & mask, pte) != pte); - - mask = 0; - } - } - private ulong AddressToOffset(ulong address) { if (address < ReservedSize) @@ -839,10 +113,7 @@ namespace Ryujinx.Cpu.Nce protected override void Destroy() { _addressSpace.Dispose(); - _addressSpaceMirror.Dispose(); _memoryEh.Dispose(); } - - private static void ThrowInvalidMemoryRegionException(string message) => throw new InvalidMemoryRegionException(message); } } diff --git a/src/Ryujinx.HLE/HOS/ArmProcessContextFactory.cs b/src/Ryujinx.HLE/HOS/ArmProcessContextFactory.cs index 41e6bad19..96a6c4db3 100644 --- a/src/Ryujinx.HLE/HOS/ArmProcessContextFactory.cs +++ b/src/Ryujinx.HLE/HOS/ArmProcessContextFactory.cs @@ -57,7 +57,6 @@ namespace Ryujinx.HLE.HOS public IProcessContext Create(KernelContext context, ulong pid, ulong addressSpaceSize, InvalidAccessHandler invalidAccessHandler, bool for64Bit) { IArmProcessContext processContext; - AddressSpace addressSpace = null; bool isArm64Host = RuntimeInformation.ProcessArchitecture == Architecture.Arm64; @@ -71,16 +70,16 @@ namespace Ryujinx.HLE.HOS } else { - if (!AddressSpace.TryCreate(context.Memory, addressSpaceSize, MemoryBlock.GetPageSize() == MemoryManagerHostMapped.PageSize, out addressSpace)) + if (!AddressSpace.TryCreateWithoutMirror(addressSpaceSize, out var addressSpace)) { throw new Exception("Address space creation failed"); } - Logger.Info?.Print(LogClass.Cpu, $"NCE Base AS Address: 0x{addressSpace.Base.Pointer.ToInt64():X} Size: 0x{addressSpace.AddressSpaceSize:X}"); + Logger.Info?.Print(LogClass.Cpu, $"NCE Base AS Address: 0x{addressSpace.Pointer.ToInt64():X} Size: 0x{addressSpace.Size:X}"); var cpuEngine = new NceEngine(_tickSource); var memoryManager = new MemoryManagerNative(addressSpace, context.Memory, addressSpaceSize, invalidAccessHandler); - processContext = new ArmProcessContext(pid, cpuEngine, _gpu, memoryManager, addressSpace.AddressSpaceSize, for64Bit, memoryManager.ReservedSize); + processContext = new ArmProcessContext(pid, cpuEngine, _gpu, memoryManager, addressSpace.Size, for64Bit, memoryManager.ReservedSize); } } else @@ -98,9 +97,13 @@ namespace Ryujinx.HLE.HOS ? new LightningJitEngine(_tickSource) : new JitEngine(_tickSource); + AddressSpace addressSpace = null; + MemoryBlock asNoMirror = null; + if (mode == MemoryManagerMode.HostMapped || mode == MemoryManagerMode.HostMappedUnsafe) { - if (!AddressSpace.TryCreate(context.Memory, addressSpaceSize, MemoryBlock.GetPageSize() == MemoryManagerHostMapped.PageSize, out addressSpace)) + if (!AddressSpace.TryCreate(context.Memory, addressSpaceSize, MemoryBlock.GetPageSize() == MemoryManagerHostMapped.PageSize, out addressSpace) && + !AddressSpace.TryCreateWithoutMirror(addressSpaceSize, out asNoMirror)) { Logger.Warning?.Print(LogClass.Cpu, "Address space creation failed, falling back to software page table"); @@ -111,24 +114,36 @@ namespace Ryujinx.HLE.HOS switch (mode) { case MemoryManagerMode.SoftwarePageTable: - var memoryManager = new MemoryManager(context.Memory, addressSpaceSize, invalidAccessHandler); - processContext = new ArmProcessContext(pid, cpuEngine, _gpu, memoryManager, addressSpaceSize, for64Bit); + { + var mm = new MemoryManager(context.Memory, addressSpaceSize, invalidAccessHandler); + processContext = new ArmProcessContext(pid, cpuEngine, _gpu, mm, addressSpaceSize, for64Bit); + } break; case MemoryManagerMode.HostMapped: case MemoryManagerMode.HostMappedUnsafe: - if (addressSpaceSize != addressSpace.AddressSpaceSize) - { - Logger.Warning?.Print(LogClass.Emulation, $"Allocated address space (0x{addressSpace.AddressSpaceSize:X}) is smaller than guest application requirements (0x{addressSpaceSize:X})"); - } + bool unsafeMode = mode == MemoryManagerMode.HostMappedUnsafe; - var memoryManagerHostMapped = new MemoryManagerHostMapped(addressSpace, mode == MemoryManagerMode.HostMappedUnsafe, invalidAccessHandler); - processContext = new ArmProcessContext(pid, cpuEngine, _gpu, memoryManagerHostMapped, addressSpace.AddressSpaceSize, for64Bit); + if (addressSpace != null) + { + var mm = new MemoryManagerHostMapped(addressSpace, unsafeMode, invalidAccessHandler); + processContext = new ArmProcessContext(pid, cpuEngine, _gpu, mm, addressSpace.AddressSpaceSize, for64Bit); + } + else + { + var mm = new MemoryManagerHostNoMirror(asNoMirror, context.Memory, unsafeMode, invalidAccessHandler); + processContext = new ArmProcessContext(pid, cpuEngine, _gpu, mm, asNoMirror.Size, for64Bit); + } break; default: throw new InvalidOperationException($"{nameof(mode)} contains an invalid value: {mode}"); } + + if (addressSpaceSize != processContext.AddressSpaceSize) + { + Logger.Warning?.Print(LogClass.Emulation, $"Allocated address space (0x{processContext.AddressSpaceSize:X}) is smaller than guest application requirements (0x{addressSpaceSize:X})"); + } } DiskCacheLoadState = processContext.Initialize(_titleIdText, _displayVersion, _diskCacheEnabled, _codeAddress, _codeSize); diff --git a/src/Ryujinx.HLE/HOS/Kernel/Memory/KPageTableBase.cs b/src/Ryujinx.HLE/HOS/Kernel/Memory/KPageTableBase.cs index 77a3092c7..ad294056d 100644 --- a/src/Ryujinx.HLE/HOS/Kernel/Memory/KPageTableBase.cs +++ b/src/Ryujinx.HLE/HOS/Kernel/Memory/KPageTableBase.cs @@ -213,11 +213,13 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory break; case AddressSpaceType.Addr39Bits: - if (_reservedAddressSpaceSize < addrSpaceEnd) - { - int addressSpaceWidth = (int)ulong.Log2(_reservedAddressSpaceSize); + ulong reservedAddressSpaceSize = _reservedAddressSpaceSize; - aliasRegion.Size = 1UL << (addressSpaceWidth - 3); + if (reservedAddressSpaceSize < addrSpaceEnd) + { + int addressSpaceWidth = (int)ulong.Log2(reservedAddressSpaceSize); + + aliasRegion.Size = reservedAddressSpaceSize >= 0x1800000000 ? 0x1000000000 : 1UL << (addressSpaceWidth - 3); heapRegion.Size = 0x180000000; stackRegion.Size = 1UL << (addressSpaceWidth - 8); tlsIoRegion.Size = 1UL << (addressSpaceWidth - 3); @@ -226,7 +228,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Memory stackAndTlsIoStart = 0; stackAndTlsIoEnd = 0; AslrRegionStart = Math.Max(reservedSize, 0x8000000); - addrSpaceEnd = reservedSize + (1UL << addressSpaceWidth); + addrSpaceEnd = reservedSize + reservedAddressSpaceSize; AslrRegionEnd = addrSpaceEnd; } else